Index: memory_optimization/lib/fastutil.jar
===================================================================
Cannot display: file marked as a binary type.
svn:mime-type = application/octet-stream
Index: memory_optimization/src/uk/me/parabola/splitter/AbstractXppParser.java
===================================================================
--- memory_optimization/src/uk/me/parabola/splitter/AbstractXppParser.java	(revision 183)
+++ memory_optimization/src/uk/me/parabola/splitter/AbstractXppParser.java	(working copy)
@@ -42,6 +42,10 @@
 	protected int getIntAttr(String name) {
 		return Integer.parseInt(parser.getAttributeValue(null, name));
 	}
+	
+	protected long getLongAttr(String name) {
+		return Long.parseLong(parser.getAttributeValue(null, name));
+	}
 
 	protected String getTextContent() {
 		return parser.getText();
Index: memory_optimization/src/uk/me/parabola/splitter/args/SplitterParams.java
===================================================================
--- memory_optimization/src/uk/me/parabola/splitter/args/SplitterParams.java	(revision 183)
+++ memory_optimization/src/uk/me/parabola/splitter/args/SplitterParams.java	(working copy)
@@ -72,4 +72,8 @@
 	
 	@Option(defaultValue = "unset", description = "The output type, either pbf or xml.")
 	String getOutput();
+
+	@Option(description = "Store data in HashMap instead of array. Use this option when heap is small.")
+	boolean isOptimizeMem();
+
 }
Index: memory_optimization/src/uk/me/parabola/splitter/BinaryMapParser.java
===================================================================
--- memory_optimization/src/uk/me/parabola/splitter/BinaryMapParser.java	(revision 183)
+++ memory_optimization/src/uk/me/parabola/splitter/BinaryMapParser.java	(working copy)
@@ -37,7 +37,8 @@
             double latf = parseLat(lat), lonf = parseLon(lon);
 			
 			Node tmp = new Node();
-			tmp.set((int)id, latf, lonf);
+			tmp.set(id, latf, lonf);
+			
 			if (!processor.isStartNodeOnly()) {
 				if (nodes.getKeysValsCount() > 0) {
 					while (nodes.getKeysVals(j) != 0) {
@@ -62,7 +63,7 @@
 			long id = i.getId();
 			double latf = parseLat(i.getLat()), lonf = parseLon(i.getLon());
 
-			tmp.set((int)id, latf, lonf);
+			tmp.set(id, latf, lonf);
 
 			processor.processNode(tmp);
 			processNodes(tmp);
@@ -79,13 +80,13 @@
 
 				long last_id=0;
 				for (long j : i.getRefsList()) {
-				tmp.addRef((int)(j+last_id));
+					tmp.addRef(j+last_id);
 					last_id = j+last_id;
 				}
 			}
 
 			long id = i.getId();
-			tmp.set((int)id);
+			tmp.set(id);
 
 			processor.processWay(tmp);
 			processWays(tmp);
@@ -99,7 +100,7 @@
 				tmp.addTag(getStringById(i.getKeys(j)),getStringById(i.getVals(j)));
 
 			long id = i.getId();
-			tmp.set((int)id);
+			tmp.set(id);
 			
 			long last_mid=0;
 			for (int j =0; j < i.getMemidsCount() ; j++) {
@@ -117,7 +118,7 @@
 				else
 					assert false; // TODO; Illegal file?
 
-				tmp.addMember(etype,(int)mid,role);
+				tmp.addMember(etype,mid,role);
 			}
 			processor.processRelation(tmp);
 			processRelations(tmp);
@@ -172,4 +173,7 @@
 		}
 	}
 
+	public long getNodeCount() {
+		return nodeCount;
+	}
 }
Index: memory_optimization/src/uk/me/parabola/splitter/BinaryMapWriter.java
===================================================================
--- memory_optimization/src/uk/me/parabola/splitter/BinaryMapWriter.java	(revision 183)
+++ memory_optimization/src/uk/me/parabola/splitter/BinaryMapWriter.java	(working copy)
@@ -255,7 +255,7 @@
           Osmformat.Way.Builder bi = Osmformat.Way.newBuilder();
           bi.setId(i.getId());
           long lastid = 0;
-          for(int j : i.getRefs()) {
+          for(long j : i.getRefs()) {
             long id = j;
             bi.addRefs(id - lastid);
             lastid = id;
Index: memory_optimization/src/uk/me/parabola/splitter/DensityMapCollector.java
===================================================================
--- memory_optimization/src/uk/me/parabola/splitter/DensityMapCollector.java	(revision 183)
+++ memory_optimization/src/uk/me/parabola/splitter/DensityMapCollector.java	(working copy)
@@ -21,7 +21,8 @@
 	private final DensityMap densityMap;
 	private final MapDetails details = new MapDetails();
 	private Area bounds;
-
+	private long maxNodeId = Long.MIN_VALUE;
+	
 	DensityMapCollector(boolean trim, int resolution) {
 		this(null, trim, resolution);
 	}
@@ -49,6 +50,9 @@
 
 	@Override
 	public void processNode(Node n) {
+		if (n.getId()> maxNodeId) {
+			maxNodeId = n.getId();
+		} 
 		int glat = n.getMapLat();
 		int glon = n.getMapLon();
 		densityMap.addNode(glat, glon);
@@ -77,4 +81,10 @@
 	public SplittableArea getRoundedArea(int resolution) {
 		Area bounds = RoundingUtils.round(getExactArea(), resolution);
 		return new SplittableDensityArea(densityMap.subset(bounds));
-	}}
\ No newline at end of file
+	}
+
+	@Override
+	public long getMaxNodeId() {
+		return maxNodeId; 
+	}
+}
\ No newline at end of file
Index: memory_optimization/src/uk/me/parabola/splitter/Element.java
===================================================================
--- memory_optimization/src/uk/me/parabola/splitter/Element.java	(revision 183)
+++ memory_optimization/src/uk/me/parabola/splitter/Element.java	(working copy)
@@ -14,23 +14,21 @@
 
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.Iterator;
-import java.util.Map;
 
 /**
  * @author Steve Ratcliffe
  */
 public class Element {
 	private ArrayList<Tag> tags; 
-	private int id;
+	private long id;
 
 	
-	protected void setId(int id) {
+	protected void setId(long id) {
 		this.id = id;
 	}
 
-	public int getId() {
+	public long getId() {
 		return id;
 	}
 
Index: memory_optimization/src/uk/me/parabola/splitter/Main.java
===================================================================
--- memory_optimization/src/uk/me/parabola/splitter/Main.java	(revision 183)
+++ memory_optimization/src/uk/me/parabola/splitter/Main.java	(working copy)
@@ -100,6 +100,11 @@
 	private int maxThreads;
 	// The output type
 	private boolean pbfOutput;
+	// The memory usage strategy 
+	private boolean optimizeMem = false;
+	
+	private long nodeCount;
+	private long maxNodeId;
 
 	public static void main(String[] args) {
 
@@ -258,6 +263,7 @@
 			System.out.println("WARNING: Specifying --legacy-split will cause the first stage of the split to take much more memory! This option is considered deprecated and will be removed in a future build.");
 		}
 
+		optimizeMem = params.isOptimizeMem();
 		maxThreads = params.getMaxThreads().getCount();
 		filenames = parser.getAdditionalParams();
 		
@@ -294,7 +300,9 @@
 		System.out.println("in " + filenames.size() + (filenames.size() == 1 ? " file" : " files"));
 
 		//System.out.println("Min node ID = " + mapReader.getMinNodeId());
-		//System.out.println("Max node ID = " + mapReader.getMaxNodeId());
+		maxNodeId = nodes.getMaxNodeId(); 
+		System.out.println("Max node ID = " + maxNodeId);
+		System.out.println("NodeCount: " + Utils.format(nodeCount));
 
 		System.out.println("Time: " + new Date());
 
@@ -364,7 +372,7 @@
 							" areas (" + areas.get(i * areasPerPass).getMapId() + " to " +
 							areas.get(i * areasPerPass + currentWriters.length - 1).getMapId() + ')');
 
-			MapProcessor processor = new SplitProcessor(currentWriters, maxThreads);
+			MapProcessor processor = new SplitProcessor(currentWriters, maxThreads, nodeCount, maxNodeId, optimizeMem);
 			processMap(processor);
 			//System.out.println("Wrote " + Utils.format(mapReader.getNodeCount()) + " nodes, " +
 			//				Utils.format(mapReader.getWayCount()) + " ways, " +
@@ -375,6 +383,7 @@
 	private void processMap(MapProcessor processor) throws XmlPullParserException {
 		// Create both an XML reader and a binary reader, Dispatch each input to the
 		// Appropriate parser.
+		
 		OSMParser parser = new OSMParser(processor, mixed);
 		if (useStdIn) {
 			System.out.println("Reading osm data from stdin...");
@@ -389,6 +398,7 @@
 			} catch (IOException e) {
 				e.printStackTrace();
 			}
+			nodeCount = parser.getNodeCount();
 		}
 
 		for (String filename : filenames) {
@@ -397,13 +407,15 @@
 				if (filename.endsWith(".osm.pbf")) {
 					// Is it a binary file?
 					File file = new File(filename);
+					BinaryMapParser tmpParser = new BinaryMapParser(processor); 
 					BlockInputStream blockinput = (new BlockInputStream(
-							new FileInputStream(file), new BinaryMapParser(processor)));
+							new FileInputStream(file), tmpParser ));
 					try {
 						blockinput.process();
 					} finally {
 						blockinput.close();
 					}
+					nodeCount += tmpParser .getNodeCount();
 				} else {
 					// No, try XML.
 					Reader reader = Utils.openFile(filename, maxThreads > 1);
@@ -413,6 +425,7 @@
 					} finally {
 						reader.close();
 					}
+					nodeCount  += parser.getNodeCount();
 				}
 			} catch (FileNotFoundException e) {
 				e.printStackTrace();
Index: memory_optimization/src/uk/me/parabola/splitter/MapCollector.java
===================================================================
--- memory_optimization/src/uk/me/parabola/splitter/MapCollector.java	(revision 183)
+++ memory_optimization/src/uk/me/parabola/splitter/MapCollector.java	(working copy)
@@ -19,4 +19,6 @@
 	SplittableArea getRoundedArea(int resolution);
 
 	void boundTag(Area bounds);
+	
+	long getMaxNodeId();
 }
Index: memory_optimization/src/uk/me/parabola/splitter/MapReader.java
===================================================================
--- memory_optimization/src/uk/me/parabola/splitter/MapReader.java	(revision 183)
+++ memory_optimization/src/uk/me/parabola/splitter/MapReader.java	(working copy)
@@ -14,13 +14,13 @@
 package uk.me.parabola.splitter;
 
 public interface MapReader {
-	//long getNodeCount();
+	long getNodeCount();
 
 	//long getWayCount();
 
 	//long getRelationCount();
 
-	//int getMinNodeId();
+	// int getMinNodeId();
 
-	//int getMaxNodeId();
+	// int getMaxNodeId();
 }
Index: memory_optimization/src/uk/me/parabola/splitter/Node.java
===================================================================
--- memory_optimization/src/uk/me/parabola/splitter/Node.java	(revision 183)
+++ memory_optimization/src/uk/me/parabola/splitter/Node.java	(working copy)
@@ -21,7 +21,7 @@
 	private double lat, lon;
 	private int mapLat, mapLon;
 
-	public void set(int id, double lat, double lon) {
+	public void set(long id, double lat, double lon) {
 		setId(id);
 		this.lat = lat;
 		this.lon = lon;
Index: memory_optimization/src/uk/me/parabola/splitter/Node2AreaMap.java
===================================================================
--- memory_optimization/src/uk/me/parabola/splitter/Node2AreaMap.java	(revision 183)
+++ memory_optimization/src/uk/me/parabola/splitter/Node2AreaMap.java	(working copy)
@@ -1,102 +1,107 @@
-/*
- * Copyright (c) 2011.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 3 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- */
-
-package uk.me.parabola.splitter;
-
-import it.unimi.dsi.fastutil.objects.ObjectArrayList;
-import java.util.BitSet;
-import java.util.HashMap;
-
-// a map that stores all areas in which a node or way occurs 
-public class Node2AreaMap {
-	private int size;
-	private final short unassigned;
-
-	private SparseInt2ShortMapInline map;   
-	private final ObjectArrayList<BitSet> dictionary; 
-	private final HashMap<BitSet, Short> index;
-
-	
-	public Node2AreaMap(short unassigned, int numareas) {
-		System.out.println("Making Node2AreaMap");
-		map = new SparseInt2ShortMapInline ();
-		this.unassigned = unassigned;
-		map.defaultReturnValue(unassigned);
-		dictionary = new ObjectArrayList<BitSet>();
-		index = new HashMap<BitSet, Short>();
-		
-		// init the dictionary
-		for (short i=0;i <= numareas; i++){
-			BitSet b = new BitSet();
-			b.set (i);
-			dictionary.add(b);
-			index.put( b, i);
-		}
-	}
-	
-	public void addTo(int key, BitSet out) {
-		int idx = map.get (key);
-		if (idx == unassigned)
-			return;
-		out.or(dictionary.get(idx));
-		//System.out.println(key+":"+out.toString());
-}
-
-	
-	public void put(int key, short val) {
-		//System.out.println("p"+key+","+val);
-
-     
-		int idx = map.get (key);
-		BitSet bnew;
-		if (idx == unassigned) {
-			size++;
-			if (size %1000000 == 0) {
-				stats();
-			}
-			// node is new -> val is equal index
-			map.put (key,val); 
-		}
-		else {
-			// unlikely: node (or way) belongs to multiple areas
-			bnew = (BitSet) dictionary.get(idx).clone();
-			bnew.set(val);
-			
-			Short combiIndex = index.get(bnew);
-			if (combiIndex == null){
-				// this is a new conbination of areas, create new entry 
-				// in the dictionary
-				combiIndex = (short) dictionary.size();
-				dictionary.add(bnew);
-				index.put(bnew, combiIndex);
-				//System.out.println("new combination " + combiIndex + " " + bnew.toString());
-			}
-			map.put (key, (short) combiIndex);
-		}
-		
-	}
-	
-	public boolean contains (int key) {
-		return map.containsKey(key);
-	}
-	
-	
-	public int size() {
-		return size;
-	}
-	
-	public void stats() {
-		System.out.println("MAP occupancy: " + Utils.format(size) + ", number of area dictionary entries: " + dictionary.size());
-		//map.stats();
-	}
-}
+/*
+ * Copyright (c) 2011.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 3 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ */
+
+package uk.me.parabola.splitter;
+
+import it.unimi.dsi.fastutil.objects.ObjectArrayList;
+import java.util.BitSet;
+import java.util.HashMap;
+
+/// a map that stores all areas in which a node occurs.
+public class Node2AreaMap {
+	private int size;
+	private final short unassigned = -1;
+
+	private SparseLong2ShortMapFunction map;
+	private final ObjectArrayList<BitSet> dictionary; 
+	private final HashMap<BitSet, Short> index;
+	
+	public Node2AreaMap(int numareas, long nodeCount, long maxNodeId, boolean optimizeMem) { 
+		if (optimizeMem) {
+			System.out.println("Making Node2AreaMap using SparseLong2ShortMapInline for (estimated) " + nodeCount + " Nodes"); 
+			map = new SparseLong2ShortMapInline (nodeCount, maxNodeId);
+		}
+		else { 
+			System.out.println("Making Node2AreaMap using SparseLong2ShortMapFix for highest NodeId " + maxNodeId); 
+			map = new SparseLong2ShortMapFix (nodeCount, maxNodeId);
+		}
+		
+
+		map.defaultReturnValue(unassigned);
+		dictionary = new ObjectArrayList<BitSet>();
+		index = new HashMap<BitSet, Short>();
+		
+		// init the dictionary
+		for (short i=0;i <= numareas; i++){
+			BitSet b = new BitSet();
+			b.set (i);
+			dictionary.add(b);
+			index.put( b, i);
+		}
+	}
+	
+	public void addTo(long key, BitSet out) {
+		int idx = map.get (key);
+		if (idx == unassigned)
+			return;
+		out.or(dictionary.get(idx));
+		//System.out.println(key+":"+out.toString());
+}
+
+	
+	public void put(long key, short val) {
+		//System.out.println("p"+key+","+val);
+
+		int idx = map.get (key);
+		BitSet bnew;
+		if (idx == unassigned) {
+			size++;
+			if (size %1000000 == 0) {
+				stats();
+			}
+			// node is new -> val is equal index
+			map.put (key,val); 
+		}
+		else {
+			// unlikely: node or way belongs to multiple areas
+			bnew = (BitSet) dictionary.get(idx).clone();
+			bnew.set(val);
+			
+			Short combiIndex = index.get(bnew);
+			if (combiIndex == null){
+				// this is a new combination of areas, create new entry 
+				// in the dictionary
+				combiIndex = (short) dictionary.size();
+				dictionary.add(bnew);
+				index.put(bnew, combiIndex);
+				//System.out.println("new combination " + combiIndex + " " + bnew.toString());
+			}
+			map.put (key, (short) combiIndex);
+		}
+		
+	}
+	
+	public boolean contains (long key) {
+		return map.containsKey(key);
+	}
+	
+	
+	public int size() {
+		return size;
+	}
+	
+	public void stats() {
+		System.out.println("MAP occupancy: " + Utils.format(size) + ", number of area dictionary entries: " + dictionary.size());
+		map.stats();
+	}
+}
Index: memory_optimization/src/uk/me/parabola/splitter/NodeCollector.java
===================================================================
--- memory_optimization/src/uk/me/parabola/splitter/NodeCollector.java	(revision 183)
+++ memory_optimization/src/uk/me/parabola/splitter/NodeCollector.java	(working copy)
@@ -21,6 +21,8 @@
 	private SplitIntList coords = new SplitIntList();
 	private final MapDetails details = new MapDetails();
 	private Area bounds;
+	private long maxNodeId = Long.MIN_VALUE;
+	
 
 	@Override
 	public boolean isStartNodeOnly() {
@@ -40,6 +42,9 @@
 		// Since we are rounding areas to fit on a low zoom boundary we
 		// can drop the bottom 8 bits of the lat and lon and then fit
 		// the whole lot into a single int.
+		if (n.getId()> maxNodeId) {
+			maxNodeId = n.getId();
+		} 
 		int glat = n.getMapLat();
 		int glon = n.getMapLon();
 		int coord = ((glat << 8) & 0xffff0000) + ((glon >> 8) & 0xffff);
@@ -73,4 +78,10 @@
 		coords = null;
 		return result;
 	}
+
+	@Override
+	public long getMaxNodeId() {
+		return maxNodeId; 
+	}
+	
 }
\ No newline at end of file
Index: memory_optimization/src/uk/me/parabola/splitter/OSMParser.java
===================================================================
--- memory_optimization/src/uk/me/parabola/splitter/OSMParser.java	(revision 183)
+++ memory_optimization/src/uk/me/parabola/splitter/OSMParser.java	(working copy)
@@ -43,20 +43,20 @@
 	private long nodeCount;
 	private long wayCount;
 	private long relationCount;
-	private int minNodeId = Integer.MAX_VALUE;
-	private int maxNodeId = Integer.MIN_VALUE;
+	private long minNodeId = Long.MAX_VALUE;
+	private long maxNodeId = Long.MIN_VALUE;
 
 	OSMParser(MapProcessor processor, boolean mixed) throws XmlPullParserException {
 		this.processor = processor;
 		this.startNodeOnly = processor.isStartNodeOnly();
 		this.mixed = mixed;
 	}
-	/*
 	@Override
 	public long getNodeCount() {
 		return nodeCount;
 	}
 
+	/*
 	@Override
 	public long getWayCount() {
 		return wayCount;
@@ -66,7 +66,6 @@
 	public long getRelationCount() {
 		return relationCount;
 	}
-
 	@Override
 	public int getMinNodeId() {
 		return minNodeId;
@@ -77,6 +76,7 @@
 		return maxNodeId;
 	}
 	*/
+
 	public void endMap() {
 		processor.endMap();
 	}
@@ -92,6 +92,7 @@
 			if (action != null && action.equals("delete"))
 				return false;
 			if (name.equals("node")) {
+			   ++nodeCount;
 				startNode();
 			} else if (name.equals("way")) {
 				if (!startNodeOnly)
@@ -132,7 +133,7 @@
 			return;
 		}
 
-		int id = Integer.parseInt(idStr);
+		long id = Long.parseLong(idStr);
 		double lat = Convert.parseDouble(latStr);
 		double lon = Convert.parseDouble(lonStr);
 
@@ -150,13 +151,13 @@
 
 	private void startWay() {
 		currentWay = new Way();
-		currentWay.set(getIntAttr("id"));
+		currentWay.set(getLongAttr("id"));
 		state = State.Way;
 	}
 
 	private void startRelation() {
 		currentRelation = new Relation();
-		currentRelation.set(getIntAttr("id"));
+		currentRelation.set(getLongAttr("id"));
 		state = State.Relation;
 	}
 
@@ -168,7 +169,7 @@
 
 	private void processWay(CharSequence name) {
 		if (name.equals("nd")) {
-			currentWay.addRef(getIntAttr("ref"));
+			currentWay.addRef(getLongAttr("ref"));
 		} else if (name.equals("tag")) {
 			currentWay.addTag(getAttr("k"), getAttr("v"));
 		}
@@ -179,7 +180,7 @@
 			currentRelation.addTag(getAttr("k"), getAttr("v"));
 		} else if (name.equals("member")) {
 			String type = getAttr("type");
-			int id = getIntAttr("ref");
+			long id = getLongAttr("ref");
 			String role = getAttr("role");
 			if ("node".equals(type) || "way".equals(type)) {
 				currentRelation.addMember(type, id, role);
@@ -242,7 +243,6 @@
 			if (name.equals("node")) {
 				processor.processNode(currentNode);
 				state = State.None;
-				nodeCount++;
 				if (nodeCount % NODE_STATUS_UPDATE_THRESHOLD == 0) {
 					System.out.println(Utils.format(nodeCount) + " nodes processed...");
 				}
Index: memory_optimization/src/uk/me/parabola/splitter/OSMXMLWriter.java
===================================================================
--- memory_optimization/src/uk/me/parabola/splitter/OSMXMLWriter.java	(revision 183)
+++ memory_optimization/src/uk/me/parabola/splitter/OSMXMLWriter.java	(working copy)
@@ -13,7 +13,7 @@
 
 package uk.me.parabola.splitter;
 
-import it.unimi.dsi.fastutil.ints.IntArrayList;
+import it.unimi.dsi.fastutil.longs.LongArrayList;
 
 import java.io.File;
 import java.io.FileOutputStream;
@@ -91,7 +91,7 @@
 	
 	public void write(Node node) throws IOException {
 		writeString("<node id='");
-		writeInt(node.getId());
+		writeLong(node.getId());
 		writeString("' lat='");
 		writeDouble(node.getLat());
 		writeString("' lon='");
@@ -107,12 +107,12 @@
 
 	public void write(Way way) throws IOException {
 		writeString("<way id='");
-		writeInt(way.getId());
+		writeLong(way.getId());
 		writeString("'>\n");
-		IntArrayList refs = way.getRefs();
+		LongArrayList refs = way.getRefs();
 		for (int i = 0; i < refs.size(); i++) {
 			writeString("<nd ref='");
-			writeInt(refs.get(i));
+			writeLong(refs.get(i));
 			writeString("'/>\n");
 		}
 		if (way.hasTags())
@@ -122,7 +122,7 @@
 
 	public void write(Relation rel) throws IOException {
 		writeString("<relation id='");
-		writeInt(rel.getId());
+		writeLong(rel.getId());
 		writeString("'>\n");
 		List<Relation.Member> memlist = rel.getMembers();
 		for (Relation.Member m : memlist) {
@@ -133,7 +133,7 @@
 			writeString("<member type='");
 			writeAttribute(m.getType());
 			writeString("' ref='");
-			writeInt(m.getRef());
+			writeLong(m.getRef());
 			writeString("' role='");
 			if (m.getRole() != null) {
 				writeAttribute(m.getRole());
@@ -237,6 +237,17 @@
 		index += Convert.intToString(value, charBuf, index);
 	}
 
+	
+	private void writeLong(long value) throws IOException {
+		if (value < Integer.MAX_VALUE) writeInt ((int) value);
+		else {
+			checkFlush(20);
+			StringBuilder s = new StringBuilder(Long.toString(value));
+			writeString(s.toString());
+		}
+	}
+	
+
 	private void writeChar(char value) throws IOException {
 		checkFlush(1);
 		charBuf[index++] = value;
Index: memory_optimization/src/uk/me/parabola/splitter/Relation.java
===================================================================
--- memory_optimization/src/uk/me/parabola/splitter/Relation.java	(revision 183)
+++ memory_optimization/src/uk/me/parabola/splitter/Relation.java	(working copy)
@@ -21,17 +21,17 @@
 public class Relation extends Element {
 	private final List<Member> members = new ArrayList<Member>();
 
-	public void set(int id) {
+	public void set(long id) {
 		setId(id);
 	}
-
+	
 	@Override
 	public void reset() {
 		super.reset();
 		members.clear();
 	}
 
-	public void addMember(String type, int ref, String role) {
+	public void addMember(String type, long ref, String role) {
 		Member mem = new Member(type, ref, role);
 		members.add(mem);
 	}
@@ -42,10 +42,10 @@
 
 	static class Member {
 		private String type;
-		private int ref;
+		private long ref;
 		private String role;
 
-		Member(String type, int ref, String role) {
+		Member(String type, long ref, String role) {
 			this.type = type;
 			this.ref = ref;
 			this.role = role;
@@ -55,7 +55,7 @@
 			return type;
 		}
 
-		public int getRef() {
+		public long getRef() {
 			return ref;
 		}
 
Index: memory_optimization/src/uk/me/parabola/splitter/SparseLong2ShortMapFix.java
===================================================================
--- memory_optimization/src/uk/me/parabola/splitter/SparseLong2ShortMapFix.java	(revision 0)
+++ memory_optimization/src/uk/me/parabola/splitter/SparseLong2ShortMapFix.java	(working copy)
@@ -0,0 +1,264 @@
+package uk.me.parabola.splitter;
+
+import it.unimi.dsi.bits.Fast;
+
+import java.util.Arrays;
+
+/**
+ * Stores long/short pairs, optimized for speed. 
+ * Inspired by SparseInt2ShortMapInline, but uses fixed arrays instead of ArrayList.
+ * Key values are limited to a maximum of 2^37 - 1.
+ * 
+ * @author GerdP
+ *
+ */
+public class SparseLong2ShortMapFix implements SparseLong2ShortMapFunction {
+	static final int POOL_SIZE = 8191;
+	private final long maxNodeId;
+	private final long nodeCount;
+
+	/** What to return on unassigned indices */
+	short unassigned = -1;
+	private long [] countChunkLen;
+	final long initsize;
+	
+	private int poolIndex = 0;
+	private int countPools = 0;
+	private short[][] pool;
+
+	private long[] maskmap;
+	private short[][] chunkmap;
+
+	long size;
+
+	SparseLong2ShortMapFix(long nodeCount, long maxNodeId) {
+		this.initsize = maxNodeId/CHUNK_SIZE;
+		this.maxNodeId = maxNodeId;
+		this.nodeCount = nodeCount;
+		clear();
+	}
+
+	void arrayPush(short[] array, int index) {
+		for (int j = array.length - 1; j > index; j--)
+			array[j] = array[j - 1];
+	}
+
+	void arrayCopyFill(short[] from, short[] to) {
+		int j = 0;
+		for (; j < from.length; j++)
+			to[j] = from[j];
+		for (; j < to.length; j++)
+			to[j] = unassigned;
+	}
+
+	short[] chunkAdd(short[] array, int index, short val) {
+		if (array[array.length - 1] != unassigned) {
+			// give it back to my pool ?
+			if (array.length == 4 & poolIndex > 0)
+				pool[--poolIndex] = array;
+			//System.out.println("F:poolIndex:" + poolIndex);
+			--countChunkLen[array.length];
+			short tmp[] = new short[array.length + 4];
+			arrayCopyFill(array, tmp);
+			array = tmp;
+			++countChunkLen[array.length];
+		}
+		arrayPush(array, index);
+		array[index] = val;
+		return array;
+	}
+
+	short[] chunkMake() {
+		short[] out;
+		if (pool[poolIndex] != null){
+			out = pool[poolIndex];
+			pool[poolIndex] = null;
+			if (poolIndex < POOL_SIZE) 
+				++poolIndex;
+			else {
+				//System.out.println("pool is full :-(");
+				makeNewPool();
+			}
+		}
+		else
+			out = new short[4];
+
+		Arrays.fill(out, unassigned);
+		++countChunkLen[out.length];
+		return out;
+	}
+
+	void chunkSet(short[] array, int index, short val) {
+		array[index] = val;
+	}
+
+	short chunkGet(short[] array, int index) {
+		return array[index];
+	}
+
+	/**
+	 * Count how many of the lowest X bits in mask are set
+	 * 
+	 * @return
+	 */
+	int countUnder(long mask, int lowest) {
+		return Fast.count(mask & ((1L << lowest) - 1));
+	}
+
+	@Override
+	public boolean containsKey(long key) {
+		if (key >= MAX_KEY) {
+			throw new IllegalArgumentException(
+					"Cannot handle such key, is too high. key=" + key);
+		}
+		int chunkid = (int) (key / CHUNK_SIZE);
+		long chunkmask = maskmap[chunkid];
+
+		if (chunkmask == 0)
+			return false;
+		int chunkoffset = (int) (key % CHUNK_SIZE);
+
+		long elementmask = 1L << chunkoffset;
+		return (chunkmask & elementmask) != 0;
+	}
+
+	@Override
+	public short put(long key, short val) {
+		if (val == unassigned) {
+			throw new IllegalArgumentException(
+					"Cannot store the value that is reserved as being unassigned. val="
+							+ val);
+		}
+		if (key < 0) {
+			throw new IllegalArgumentException("Cannot store the negative key,"
+					+ key);
+		} else if (key >= MAX_KEY) {
+			throw new IllegalArgumentException(
+					"Cannot handle such key, is too high. key=" + key);
+		}
+		short chunk[];
+		int chunkid = (int) (key / CHUNK_SIZE);
+		int chunkoffset = (int) (key % CHUNK_SIZE);
+
+		long chunkmask = maskmap[chunkid];
+		if (chunkmask == 0){
+			//chunkmask = 0;
+			chunk = chunkMake();
+		}
+		else 
+			chunk = chunkmap[chunkid];
+		
+		long elementmask = 1L << chunkoffset;
+		if ((chunkmask & elementmask) != 0) {
+			// Already in the array, find the offset and store.
+			short out = chunkGet(chunk,
+					countUnder(chunkmask, chunkoffset));
+			chunkSet(chunk, countUnder(chunkmask, chunkoffset), val);
+			// System.out.println("Returning found key "+out+" from put "+ key +
+			// " " + val);
+			return out;
+		} else {
+			size++;
+			// Not in the array. Time to insert.
+			int offset = countUnder(chunkmask, chunkoffset);
+			chunk = chunkAdd(chunk, offset, val);
+			chunkmask |= elementmask;
+			maskmap[chunkid] = chunkmask;
+			chunkmap[chunkid] = chunk;
+			return unassigned;
+		}
+	}
+
+	@Override
+	public short get(long key) {
+		if (key >= MAX_KEY) {
+			throw new IllegalArgumentException(
+					"Cannot handle such key, is too high. key=" + key);
+		}
+		
+		int chunkid = (int) (key / CHUNK_SIZE);
+		long chunkmask = maskmap[chunkid];
+		if (chunkmask == 0)
+			return unassigned;
+		int chunkoffset = (int) key % CHUNK_SIZE;
+		long elementmask = 1L << chunkoffset;
+		if ((chunkmask & elementmask) == 0) {
+			return unassigned;
+		} else {
+			short [] chunk = chunkmap[chunkid];
+			return chunkGet(chunk, countUnder(chunkmask, chunkoffset));
+		}
+	}
+
+	@Override
+	public void clear() {
+		chunkmap = new short [(int)(this.maxNodeId /CHUNK_SIZE)+1][];
+		countChunkLen = new long[CHUNK_SIZE+1];
+		maskmap = new long[(int)(this.maxNodeId /CHUNK_SIZE)+1];
+		makeNewPool();
+		size = 0;
+	}
+
+	@Override
+	public short remove(long key) {
+		throw new UnsupportedOperationException("TODO: Implement");
+	}
+
+	@Override
+	public boolean containsKey(Object arg0) {
+		throw new UnsupportedOperationException("TODO: Implement");
+	}
+
+	@Override
+	public Short get(Object arg0) {
+		throw new UnsupportedOperationException("TODO: Implement");
+	}
+
+	@Override
+	public Short put(Long arg0, Short arg1) {
+		return put(arg0.intValue(), arg1.shortValue());
+	}
+
+	@Override
+	public Short remove(Object arg0) {
+		throw new UnsupportedOperationException("TODO: Implement");
+	}
+	
+	@Override
+	public int size() {
+		return (int) size; 
+	}
+	
+	@Override
+	public short defaultReturnValue() {
+		return unassigned;
+	}
+
+	@Override
+	public void defaultReturnValue(short arg0) {
+		unassigned = arg0;
+	}
+	
+	private void makeNewPool(){
+		pool = new short[POOL_SIZE+1][4];
+		poolIndex = 0;
+		++countPools;
+	}
+
+	@Override
+	public void stats() {
+		long usedChunks = 0;
+		long pctusage = 0L;
+		for (int i=4; i<=CHUNK_SIZE; i+=4) {
+			usedChunks += countChunkLen[i];
+			if (countChunkLen[i] > 0) {
+				//	System.out.println("type-" + i + " chunks: " + Utils.format(countChunkLen[i]));
+			}
+		}
+		if (maskmap.length > 0) 
+			pctusage = Math.round((double)usedChunks*100/maskmap.length) ;
+
+		System.out.println("chunk map details: Used " + Utils.format(usedChunks) + " of " + Utils.format(maskmap.length) + " allocated map entries (~ " + pctusage + "%)");
+	}
+}	
+
Index: memory_optimization/src/uk/me/parabola/splitter/SparseLong2ShortMapFunction.java
===================================================================
--- memory_optimization/src/uk/me/parabola/splitter/SparseLong2ShortMapFunction.java	(revision 0)
+++ memory_optimization/src/uk/me/parabola/splitter/SparseLong2ShortMapFunction.java	(working copy)
@@ -0,0 +1,18 @@
+package uk.me.parabola.splitter;
+
+import it.unimi.dsi.fastutil.longs.Long2ShortFunction;
+
+/**
+ * Stores long/short pairs. 
+ * Inspired by SparseInt2ShortMapInline.
+ * Key values are limited to a maximum of 2^37 - 1.
+ * 
+ * @author GerdP
+ *
+ */
+interface SparseLong2ShortMapFunction extends Long2ShortFunction {
+	static final int CHUNK_SIZE = 64; // MUST be <= 64.
+	static final long MAX_KEY = (long) (CHUNK_SIZE * (long) Integer.MIN_VALUE * -1L) - 1;  // 2^37 - 1 
+	
+	public void stats();
+}
Index: memory_optimization/src/uk/me/parabola/splitter/SparseLong2ShortMapInline.java
===================================================================
--- memory_optimization/src/uk/me/parabola/splitter/SparseLong2ShortMapInline.java	(revision 0)
+++ memory_optimization/src/uk/me/parabola/splitter/SparseLong2ShortMapInline.java	(working copy)
@@ -0,0 +1,277 @@
+package uk.me.parabola.splitter;
+
+import it.unimi.dsi.bits.Fast;
+import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
+
+/**
+ * SparseLong2ShortMapFunction optimized for low memory requirements.
+ * 
+ * @author GerdP
+ *
+ */
+public class SparseLong2ShortMapInline implements SparseLong2ShortMapFunction{
+	static final int MASK_SIZE = 4;	// number of chunk elements used to store mask
+	static final int POOL_SIZE = 8191;
+	
+	/** What to return on unassigned indices */
+	short unassigned = -1;
+	private long [] countChunkLen;
+	final int initsize;
+	
+	// we use a simple pool to be able to reuse allocated small chunks
+	private short[][] pool;
+	private int poolIndex = 0;
+	
+	private Int2ObjectOpenHashMap<short[]> chunkmap;
+
+	int size;
+
+	SparseLong2ShortMapInline(long nodeCount, long MaxNodeId) {
+		// estimate the needed elements
+		this.initsize = (int) (nodeCount/(CHUNK_SIZE/8));
+		clear();
+		
+	}
+
+	void arrayPush(short[] array, int index) {
+		for (int j = array.length - 1; j > index + MASK_SIZE; j--)
+			array[j] = array[j - 1];
+	}
+
+	void arrayCopyFill(short[] from, short[] to) {
+		int j = 0;
+		for (; j < from.length; j++)
+			to[j] = from[j];
+		for (; j < to.length; j++)
+			to[j] = unassigned;
+	}
+
+	short[] chunkAdd(short[] array, int index, short val) {
+		if (array[array.length - 1] != unassigned) {
+			if (array.length == 8 & poolIndex > 0)
+				pool[--poolIndex] = array; // move chunk back into the pool
+			--countChunkLen[array.length];
+			short tmp[] = new short[array.length + 4];
+			arrayCopyFill(array, tmp);
+			array = tmp;
+			++countChunkLen[array.length];
+		}
+		arrayPush(array, index);
+		array[index + MASK_SIZE] = val;
+		return array;
+	}
+
+	short[] chunkMake() {
+		short[] out;
+
+		if (pool[poolIndex] != null){
+			out = pool[poolIndex];
+			storeMask(0L, out);
+			pool[poolIndex] = null;
+			if (poolIndex < POOL_SIZE) 
+				++poolIndex;
+			else {
+				// chunks from current pool are now all in use by the chunkmap. 
+				makeNewPool();
+			}
+		}
+		else
+			out = new short[8];
+		/* fill last for bytes , the first 4 bytes are used for chunkmask */
+		for (int j = out.length-1; j >= MASK_SIZE; j--)
+			out[j] = unassigned; 
+		++countChunkLen[out.length];
+		return out;
+	}
+
+	void chunkSet(short[] array, int index, short val) {
+		array[index + MASK_SIZE] = val;
+	}
+
+	short chunkGet(short[] array, int index) {
+		assert index+MASK_SIZE < array.length;
+		return array[index + MASK_SIZE];
+	}
+
+
+	/**
+	 * Count how many of the lowest X bits in mask are set
+	 * 
+	 * @return
+	 */
+	int countUnder(long mask, int lowest) {
+		return Fast.count(mask & ((1L << lowest) - 1));
+	}
+
+	@Override
+	public boolean containsKey(long key) {
+		if (key >= MAX_KEY) {
+			throw new IllegalArgumentException(
+					"Cannot handle such key, is too high. key=" + key);
+		}
+		int chunkid = (int) (key / CHUNK_SIZE);
+		short [] chunk = chunkmap.get(chunkid);
+		if (chunk == null) 
+			return false;
+		long chunkmask = extractMask(chunk);
+		int chunkoffset = (int) (key % CHUNK_SIZE);
+
+		long elementmask = 1L << chunkoffset;
+		return (chunkmask & elementmask) != 0;
+	}
+
+	@Override
+	public short put(long key, short val) {
+		if (val == unassigned) {
+			throw new IllegalArgumentException(
+					"Cannot store the value that is reserved as being unassigned. val="
+							+ val);
+		}
+		if (key < 0) {
+			throw new IllegalArgumentException("Cannot store the negative key,"
+					+ key);
+		} else if (key >= MAX_KEY) {
+			throw new IllegalArgumentException(
+					"Cannot handle such key, is too high. key=" + key);
+		}
+
+		int chunkid = (int) (key / CHUNK_SIZE);
+		int chunkoffset = (int) (key % CHUNK_SIZE);
+		short [] chunk = chunkmap.get(chunkid);
+		if (chunk == null)
+			chunk = chunkMake();
+		
+		long chunkmask = extractMask(chunk);
+		
+		long elementmask = 1L << chunkoffset;
+		if ((chunkmask & elementmask) != 0) {
+			// Already in the array, find the offset and store.
+			short out = chunkGet(chunk,
+					countUnder(chunkmask, chunkoffset));
+			chunkSet(chunk, countUnder(chunkmask, chunkoffset), val);
+			// System.out.println("Returning found key "+out+" from put "+ key +
+			// " " + val);
+			return out;
+		} else {
+			size++;
+			// Not in the array. Time to insert.
+			int offset = countUnder(chunkmask, chunkoffset);
+			chunk = chunkAdd(chunk, offset, val);
+			chunkmask |= elementmask;
+			storeMask(chunkmask, chunk);
+			chunkmap.put(chunkid, chunk);
+			return unassigned;
+		}
+	}
+	
+	@Override
+	public short get(long key) {
+		if (key >= MAX_KEY) {
+			throw new IllegalArgumentException(
+					"Cannot handle such key, is too high. key=" + key);
+		}
+
+		int chunkid = (int) (key / CHUNK_SIZE);
+		short [] chunk = chunkmap.get(chunkid);
+		if (chunk == null) 
+			return unassigned;
+		long chunkmask = extractMask(chunk);
+		int chunkoffset = (int) key % CHUNK_SIZE;
+		long elementmask = 1L << chunkoffset;
+		if ((chunkmask & elementmask) == 0) {
+			return unassigned;
+		} else {
+			return chunkGet(chunk, countUnder(chunkmask, chunkoffset));
+		}
+	}
+
+	@Override
+	public void clear() {
+		chunkmap = new Int2ObjectOpenHashMap<short[]>(initsize);
+		countChunkLen = new long[CHUNK_SIZE + MASK_SIZE + 1 ];
+		makeNewPool();
+		size = 0;
+	}
+
+	@Override
+	public short remove(long key) {
+		throw new UnsupportedOperationException("TODO: Implement");
+	}
+
+
+	@Override
+	public boolean containsKey(Object arg0) {
+		throw new UnsupportedOperationException("TODO: Implement");
+	}
+
+	@Override
+	public Short get(Object arg0) {
+		throw new UnsupportedOperationException("TODO: Implement");
+	}
+
+	@Override
+	public Short put(Long arg0, Short arg1) {
+		return put(arg0.intValue(), arg1.shortValue());
+	}
+
+	@Override
+	public Short remove(Object arg0) {
+		throw new UnsupportedOperationException("TODO: Implement");
+	}
+
+	@Override
+	public int size() {
+		return size;
+	}
+
+	@Override
+	public short defaultReturnValue() {
+		return unassigned;
+	}
+
+	@Override
+	public void defaultReturnValue(short arg0) {
+		unassigned = arg0;
+	}
+	
+	private void storeMask (long mask, short[] chunk) {
+			// store chunkmask in chunk
+			long tmp = mask;
+			if (tmp == 32768)
+				tmp = 32768;
+			for (int i = 0; i < MASK_SIZE; i++){ 
+				chunk[i] = (short) (tmp & 0xffffL);
+				tmp = (tmp >> 16);
+	}
+			}
+	private long extractMask(short [] chunk){
+		long mask = 0;
+		for (int i = 0; i < MASK_SIZE; i++) {
+			mask |= (chunk[3-i] & 0xffff);
+			if (i <= 2)
+				mask = (mask << 16);
+		}
+		return mask;
+	}
+
+	private void makeNewPool(){
+		pool = new short[POOL_SIZE+1][8];
+		poolIndex = 0;
+	}
+	
+	@Override
+	public void stats() {
+		long usedChunks = 0;
+		long pctusage = 0;
+		for (int i=4; i <=CHUNK_SIZE + MASK_SIZE; i+=4) {
+			usedChunks += countChunkLen[i];
+			if (countChunkLen[i] > 0) {
+				//	System.out.println("type-" + i + " chunks: " + Utils.format(countChunkLen[i]));
+			}
+		}
+		if (initsize > 0) 
+			pctusage = Math.round((double)usedChunks*100/initsize) ;
+		System.out.println("chunk map details: Used " + Utils.format(chunkmap.size()) + " (~ " + pctusage + "%) of the estimated " + initsize + " map entries." );
+	}
+}	
+
Index: memory_optimization/src/uk/me/parabola/splitter/SplitProcessor.java
===================================================================
--- memory_optimization/src/uk/me/parabola/splitter/SplitProcessor.java	(revision 183)
+++ memory_optimization/src/uk/me/parabola/splitter/SplitProcessor.java	(working copy)
@@ -33,14 +33,14 @@
 	private final Node2AreaMap ways;
 	private final Node2AreaMap coords;
 	
-
+	
 	private final OSMWriter[] writers;
 	private final InputQueueInfo[] writerInputQueues;
 	private final BlockingQueue<InputQueueInfo> toProcess;
 	private final ArrayList<Thread> workerThreads;
 	private final InputQueueInfo STOP_MSG = new InputQueueInfo(null);
 
-	private int currentNodeAreaSet;
+	//private int currentNodeAreaSet;
 	private BitSet currentWayAreaSet;
 	private BitSet currentRelAreaSet;
 	
@@ -82,10 +82,10 @@
 		}
 	}
 
-	SplitProcessor(OSMWriter[] writers, int maxThreads) {
+	SplitProcessor(OSMWriter[] writers, int maxThreads, long nodeCount, long maxNodeId, boolean optimizeMem) {
 		this.writers = writers;
-		this.ways = new Node2AreaMap((short) -1, writers.length);
-		this.coords = new Node2AreaMap((short) -1, writers.length);
+		this.coords = new Node2AreaMap(writers.length, nodeCount, maxNodeId, optimizeMem);
+		this.ways = new Node2AreaMap(writers.length, nodeCount/4, maxNodeId, optimizeMem); 
 		makeWriterMap();
 		this.maxThreads = maxThreads;
 		this.toProcess = new ArrayBlockingQueue<InputQueueInfo>(writers.length); 
@@ -119,7 +119,7 @@
 	public void processNode(Node n) {
 		try {
 			writeNode(n);
-			currentNodeAreaSet = 0;
+			//currentNodeAreaSet = 0;
 		} catch (IOException e) {
 			throw new RuntimeException("failed to write node " + n.getId(), e);
 		}
@@ -128,7 +128,7 @@
 	@Override
 	public void processWay(Way w) {
 
-		for (int id: w.getRefs()) {
+		for (long id: w.getRefs()) {
 			// Get the list of areas that the node is in.  A node may be in
 			// more than one area because of overlap.
 			coords.addTo(id, currentWayAreaSet);
@@ -145,8 +145,8 @@
 	public void processRelation(Relation r) {
 		try {
 			for (Member mem : r.getMembers()) {
-				String role = mem.getRole();
-				int id = mem.getRef();
+				//String role = mem.getRole();
+				long id = mem.getRef();
 				if (mem.getType().equals("node")) {
 					coords.addTo(id, currentRelAreaSet);
 				} else if (mem.getType().equals("way")) {
@@ -342,4 +342,4 @@
 			System.out.println("Thread " + Thread.currentThread().getName() + " has finished");
 		}
 	}
-}
\ No newline at end of file
+}
Index: memory_optimization/src/uk/me/parabola/splitter/Way.java
===================================================================
--- memory_optimization/src/uk/me/parabola/splitter/Way.java	(revision 183)
+++ memory_optimization/src/uk/me/parabola/splitter/Way.java	(working copy)
@@ -12,15 +12,15 @@
  */
 package uk.me.parabola.splitter;
 
-import it.unimi.dsi.fastutil.ints.IntArrayList;
+import it.unimi.dsi.fastutil.longs.LongArrayList;
 
 /**
  * @author Steve Ratcliffe
  */
 public class Way extends Element {
-	private final IntArrayList refs = new IntArrayList(10);
+	private final LongArrayList refs = new LongArrayList(10);
 
-	public void set(int id) {
+	public void set(long id) {
 		setId(id);
 	}
 
@@ -30,11 +30,11 @@
 		refs.clear();
 	}
 
-	public void addRef(int ref) {
+	public void addRef(long ref) {
 		refs.add(ref);
 	}
 
-	public IntArrayList getRefs() {
+	public LongArrayList getRefs() {
 		return refs;
 	}
 }
