Index: memory_optimization/src/uk/me/parabola/splitter/args/SplitterParams.java
===================================================================
--- memory_optimization/src/uk/me/parabola/splitter/args/SplitterParams.java	(revision 187)
+++ memory_optimization/src/uk/me/parabola/splitter/args/SplitterParams.java	(working copy)
@@ -28,7 +28,7 @@
 	@Option(description = "A default description to give to each area.")
 	String getDescription();
 
-	@Option(defaultValue = "255", description = "The maximum number of areas to process in a single pass. More areas require more memory. 1-255.")
+	@Option(defaultValue = "1024", description = "The maximum number of areas to process in a single pass. More areas require more memory. 1-1024.")
 	int getMaxAreas();
 
 	@Option(defaultValue = "2000", description = "Nodes/ways/rels that fall outside an area will still be included if they are within this many map units.")
Index: memory_optimization/src/uk/me/parabola/splitter/BinaryMapParser.java
===================================================================
--- memory_optimization/src/uk/me/parabola/splitter/BinaryMapParser.java	(revision 187)
+++ memory_optimization/src/uk/me/parabola/splitter/BinaryMapParser.java	(working copy)
@@ -198,7 +198,4 @@
 		}
 	}
 
-	public long getNodeCount() {
-		return nodeCount;
-	}
 }
Index: memory_optimization/src/uk/me/parabola/splitter/DensityMapCollector.java
===================================================================
--- memory_optimization/src/uk/me/parabola/splitter/DensityMapCollector.java	(revision 187)
+++ memory_optimization/src/uk/me/parabola/splitter/DensityMapCollector.java	(working copy)
@@ -21,8 +21,7 @@
 	private final DensityMap densityMap;
 	private final MapDetails details = new MapDetails();
 	private Area bounds;
-	private long maxNodeId = Long.MIN_VALUE;
-	
+
 	DensityMapCollector(boolean trim, int resolution) {
 		this(null, trim, resolution);
 	}
@@ -50,8 +49,6 @@
 
 	@Override
 	public void processNode(Node n) {
-		if (n.getId()> maxNodeId) 
-			maxNodeId = n.getId();
 		int glat = n.getMapLat();
 		int glon = n.getMapLon();
 		densityMap.addNode(glat, glon);
@@ -59,10 +56,7 @@
 	}
 
 	@Override
-	public void processWay(Way w) {
-		if (w.getId()> maxNodeId) 
-			maxNodeId = w.getId();
-	}
+	public void processWay(Way w) {}
 
 	@Override
 	public void processRelation(Relation r) {}
@@ -84,9 +78,4 @@
 		Area bounds = RoundingUtils.round(getExactArea(), resolution);
 		return new SplittableDensityArea(densityMap.subset(bounds));
 	}
-
-	@Override
-	public long getMaxNodeId() {
-		return maxNodeId; 
-	}
 }
\ No newline at end of file
Index: memory_optimization/src/uk/me/parabola/splitter/Main.java
===================================================================
--- memory_optimization/src/uk/me/parabola/splitter/Main.java	(revision 187)
+++ memory_optimization/src/uk/me/parabola/splitter/Main.java	(working copy)
@@ -47,8 +47,9 @@
 public class Main {
 	private static final String DEFAULT_DIR = ".";
 
-	// We can only process a maximum of 255 areas at a time because we
-	// compress an area ID into 8 bits to save memory (and 0 is reserved)
+	// We store area IDs and all used combinations of area IDs in a dictionary. The index to this
+	// dictionary is saved in short values. If Short.MaxValue() is reached, the user might limit 
+	// the number of areas that is processed in one pass.  
 	private int maxAreasPerPass;
 
 	// A list of the OSM files to parse.
@@ -100,12 +101,9 @@
 	private int maxThreads;
 	// The output type
 	private boolean pbfOutput;
-	// The memory usage strategy 
+	// The memory usage strategy. Default is: favor speed by using ArrayList instead of HashMap  
 	private boolean optimizeMem = false;
 	
-	private long nodeCount;
-	private long maxNodeId;
-
 	public static void main(String[] args) {
 
 		Main m = new Main();
@@ -253,9 +251,9 @@
 		fileOutputDir = new File(outputDir == null? DEFAULT_DIR: outputDir);
 
 		maxAreasPerPass = params.getMaxAreas();
-		if (maxAreasPerPass < 1 || maxAreasPerPass > 255) {
-			System.err.println("The --max-areas parameter must be a value between 1 and 255. Resetting to 255.");
-			maxAreasPerPass = 255;
+		if (maxAreasPerPass < 1 || maxAreasPerPass > 1024) {
+			System.err.println("The --max-areas parameter must be a value between 1 and 1024. Resetting to 1024.");
+			maxAreasPerPass = 1024;
 		}
 		kmlOutputFile = params.getWriteKml();
 		densityMap = !params.isLegacyMode();
@@ -300,9 +298,7 @@
 		System.out.println("in " + filenames.size() + (filenames.size() == 1 ? " file" : " files"));
 
 		//System.out.println("Min node ID = " + mapReader.getMinNodeId());
-		maxNodeId = nodes.getMaxNodeId(); 
-		System.out.println("Max node ID = " + maxNodeId);
-		System.out.println("NodeCount: " + Utils.format(nodeCount));
+		//System.out.println("Max node ID = " + mapReader.getMaxNodeId());
 
 		System.out.println("Time: " + new Date());
 
@@ -372,7 +368,7 @@
 							" areas (" + areas.get(i * areasPerPass).getMapId() + " to " +
 							areas.get(i * areasPerPass + currentWriters.length - 1).getMapId() + ')');
 
-			MapProcessor processor = new SplitProcessor(currentWriters, maxThreads, nodeCount, maxNodeId, optimizeMem);
+			MapProcessor processor = new SplitProcessor(currentWriters, maxThreads, optimizeMem);
 			processMap(processor);
 			//System.out.println("Wrote " + Utils.format(mapReader.getNodeCount()) + " nodes, " +
 			//				Utils.format(mapReader.getWayCount()) + " ways, " +
@@ -383,7 +379,6 @@
 	private void processMap(MapProcessor processor) throws XmlPullParserException {
 		// Create both an XML reader and a binary reader, Dispatch each input to the
 		// Appropriate parser.
-		
 		OSMParser parser = new OSMParser(processor, mixed);
 		if (useStdIn) {
 			System.out.println("Reading osm data from stdin...");
@@ -398,7 +393,6 @@
 			} catch (IOException e) {
 				e.printStackTrace();
 			}
-			nodeCount = parser.getNodeCount();
 		}
 
 		for (String filename : filenames) {
@@ -407,15 +401,13 @@
 				if (filename.endsWith(".pbf")) {
 					// Is it a binary file?
 					File file = new File(filename);
-					BinaryMapParser tmpParser = new BinaryMapParser(processor); 
 					BlockInputStream blockinput = (new BlockInputStream(
-							new FileInputStream(file), tmpParser ));
+							new FileInputStream(file), new BinaryMapParser(processor)));
 					try {
 						blockinput.process();
 					} finally {
 						blockinput.close();
 					}
-					nodeCount += tmpParser .getNodeCount();
 				} else {
 					// No, try XML.
 					Reader reader = Utils.openFile(filename, maxThreads > 1);
@@ -425,7 +417,6 @@
 					} finally {
 						reader.close();
 					}
-					nodeCount  += parser.getNodeCount();
 				}
 			} catch (FileNotFoundException e) {
 				System.out.printf("ERROR: file %s was not found\n", filename);
Index: memory_optimization/src/uk/me/parabola/splitter/MapCollector.java
===================================================================
--- memory_optimization/src/uk/me/parabola/splitter/MapCollector.java	(revision 187)
+++ memory_optimization/src/uk/me/parabola/splitter/MapCollector.java	(working copy)
@@ -19,6 +19,4 @@
 	SplittableArea getRoundedArea(int resolution);
 
 	void boundTag(Area bounds);
-	
-	long getMaxNodeId();
 }
Index: memory_optimization/src/uk/me/parabola/splitter/MapReader.java
===================================================================
--- memory_optimization/src/uk/me/parabola/splitter/MapReader.java	(revision 187)
+++ memory_optimization/src/uk/me/parabola/splitter/MapReader.java	(working copy)
@@ -14,13 +14,13 @@
 package uk.me.parabola.splitter;
 
 public interface MapReader {
-	long getNodeCount();
+	//long getNodeCount();
 
 	//long getWayCount();
 
 	//long getRelationCount();
 
-	// int getMinNodeId();
+	//int getMinNodeId();
 
-	// int getMaxNodeId();
+	//int getMaxNodeId();
 }
Index: memory_optimization/src/uk/me/parabola/splitter/Node2AreaMap.java
===================================================================
--- memory_optimization/src/uk/me/parabola/splitter/Node2AreaMap.java	(revision 187)
+++ memory_optimization/src/uk/me/parabola/splitter/Node2AreaMap.java	(working copy)
@@ -27,9 +27,9 @@
 	private final HashMap<BitSet, Short> index;
 	private BitSet btst;
 
-	public Node2AreaMap(int numareas, long nodeCount, long maxNodeId, boolean optimizeMem) {
+	public Node2AreaMap(int numareas, boolean optimizeMem) {
 		System.out.println("Making Node2AreaMap");		
-		map = new SparseLong2ShortMapInline (nodeCount, maxNodeId, optimizeMem);
+		map = new SparseLong2ShortMapInline (optimizeMem);
 		map.defaultReturnValue(unassigned);
 		dictionary = new ObjectArrayList<BitSet>();
 		index = new HashMap<BitSet, Short>();
@@ -60,7 +60,7 @@
 		if (idx == unassigned) {
 			size++;
 			if (size %1000000 == 0) {
-				stats();
+				stats(0);
 			}
 			// node is new -> val is equal index
 			map.put (key,val); 
@@ -99,8 +99,8 @@
 		return size;
 	}
 
-	public void stats() {
+	public void stats(int msgLevel) {
 		System.out.println("MAP occupancy: " + Utils.format(size) + ", number of area dictionary entries: " + dictionary.size());
-		map.stats();
+		map.stats(msgLevel);
 	}
 }
Index: memory_optimization/src/uk/me/parabola/splitter/NodeCollector.java
===================================================================
--- memory_optimization/src/uk/me/parabola/splitter/NodeCollector.java	(revision 187)
+++ memory_optimization/src/uk/me/parabola/splitter/NodeCollector.java	(working copy)
@@ -21,8 +21,6 @@
 	private SplitIntList coords = new SplitIntList();
 	private final MapDetails details = new MapDetails();
 	private Area bounds;
-	private long maxNodeId = Long.MIN_VALUE;
-	
 
 	@Override
 	public boolean isStartNodeOnly() {
@@ -47,18 +45,12 @@
 		int glon = n.getMapLon();
 		int coord = ((glat << 8) & 0xffff0000) + ((glon >> 8) & 0xffff);
 
-		if (n.getId()> maxNodeId) 
-			maxNodeId = n.getId();
 		coords.add(coord);
 		details.addToBounds(glat, glon);
 	}
 
 	@Override
-	public void processWay(Way w) {
-		if (w.getId()> maxNodeId) 
-			maxNodeId = w.getId();
-		
-	}
+	public void processWay(Way w) {}
 
 	@Override
 	public void processRelation(Relation r) {}
@@ -82,10 +74,4 @@
 		coords = null;
 		return result;
 	}
-
-	@Override
-	public long getMaxNodeId() {
-		return maxNodeId; 
-	}
-	
 }
\ No newline at end of file
Index: memory_optimization/src/uk/me/parabola/splitter/OSMParser.java
===================================================================
--- memory_optimization/src/uk/me/parabola/splitter/OSMParser.java	(revision 187)
+++ memory_optimization/src/uk/me/parabola/splitter/OSMParser.java	(working copy)
@@ -51,12 +51,12 @@
 		this.startNodeOnly = processor.isStartNodeOnly();
 		this.mixed = mixed;
 	}
+	/*
 	@Override
 	public long getNodeCount() {
 		return nodeCount;
 	}
 
-	/*
 	@Override
 	public long getWayCount() {
 		return wayCount;
@@ -66,6 +66,7 @@
 	public long getRelationCount() {
 		return relationCount;
 	}
+
 	@Override
 	public int getMinNodeId() {
 		return minNodeId;
@@ -76,7 +77,6 @@
 		return maxNodeId;
 	}
 	*/
-
 	public void endMap() {
 		processor.endMap();
 	}
@@ -92,18 +92,12 @@
 			if (action != null && action.equals("delete"))
 				return false;
 			if (name.equals("node")) {
-			   ++nodeCount;
 				startNode();
 			} else if (name.equals("way")) {
 				if (!startNodeOnly)
 					startWay();
-				else {
-					long wayid =  getLongAttr("id");
-					if (wayid > maxNodeId) 
-						maxNodeId = wayid;
-					state = State.Way;					
-					if (!mixed) return true;
-				}
+				else if (!mixed)
+					return true;
 			} else if (name.equals("relation")) {
 				if (!startNodeOnly)
 					startRelation();
@@ -248,6 +242,7 @@
 			if (name.equals("node")) {
 				processor.processNode(currentNode);
 				state = State.None;
+				nodeCount++;
 				if (nodeCount % NODE_STATUS_UPDATE_THRESHOLD == 0) {
 					System.out.println(Utils.format(nodeCount) + " nodes processed...");
 				}
Index: memory_optimization/src/uk/me/parabola/splitter/SparseLong2ShortMapFunction.java
===================================================================
--- memory_optimization/src/uk/me/parabola/splitter/SparseLong2ShortMapFunction.java	(revision 187)
+++ memory_optimization/src/uk/me/parabola/splitter/SparseLong2ShortMapFunction.java	(working copy)
@@ -14,5 +14,5 @@
 	static final int CHUNK_SIZE = 64; // MUST be <= 64.
 	static final long MAX_KEY = (long) (CHUNK_SIZE * (long) Integer.MIN_VALUE * -1L) - 1;  // 2^37 - 1 
 	
-	public void stats();
+	public void stats(int msgLevel);
 }
Index: memory_optimization/src/uk/me/parabola/splitter/SparseLong2ShortMapInline.java
===================================================================
--- memory_optimization/src/uk/me/parabola/splitter/SparseLong2ShortMapInline.java	(revision 187)
+++ memory_optimization/src/uk/me/parabola/splitter/SparseLong2ShortMapInline.java	(working copy)
@@ -2,6 +2,7 @@
 
 import it.unimi.dsi.bits.Fast;
 import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
+import it.unimi.dsi.fastutil.objects.ObjectArrayList;
 
 /**
  * SparseLong2ShortMapFunction optimized for low memory requirements.
@@ -10,31 +11,28 @@
  *
  */
 public class SparseLong2ShortMapInline implements SparseLong2ShortMapFunction{
+	static final int SIZE_INCR = 1024; 
 	static final int MASK_SIZE = 4;	// number of chunk elements used to store mask
 	static final int POOL_SIZE = 8192;
 	enum Method {vector, hashmap};
 	private final Method method; 
-	private final long maxNodeId;
-	private final long nodeCount;
 
 	/** What to return on unassigned indices */
 	short unassigned = -1;
 	private long [] countChunkLen;
-	int initsize;
-
+	int capacity = 0; // available chunks in vector 
+	int initsize = 32768; // estimated number of needed chunks
+	
 	// we use a simple pool to be able to reuse allocated small chunks
 	private short[][] pool;
 	private int poolIndex = 0;
 
 	private Int2ObjectOpenHashMap<short[]> chunkmap;
-	private short[][] chunkvector;
+	private ObjectArrayList<short[]> chunkvector;
 
 	int size;
 
-	SparseLong2ShortMapInline(long nodeCount, long maxNodeId, boolean optimizeMem) {
-		// estimate the needed elements
-		this.maxNodeId = maxNodeId;
-		this.nodeCount = nodeCount;
+	SparseLong2ShortMapInline(boolean optimizeMem) {
 		if (optimizeMem)
 			method = Method.hashmap;
 		else 
@@ -197,16 +195,17 @@
 	@Override
 	public void clear() {
 		if (method == Method.hashmap) {
-			initsize = (int) (nodeCount/(CHUNK_SIZE/8)) + 1;
-			System.out.println("Allocating hashmap for (estimated) " + Utils.format(initsize) + " chunks to store " + nodeCount + " ids."); 
+			System.out.println("Allocating hashmap (Int2ObjectOpenHashMap) for chunk data"); 
 			chunkmap = new Int2ObjectOpenHashMap<short[]>(initsize);
 		}
 		else { 
-			initsize = (int) (maxNodeId /CHUNK_SIZE) + 1;
-			System.out.println("Allocating vector for " + Utils.format(initsize) + " chunks to store " + nodeCount + " ids.");
-			chunkvector = new short [(int)(this.maxNodeId /CHUNK_SIZE)+1][];
+			capacity = (int) ((1L << 31) / CHUNK_SIZE); // OSM data already contains IDs > 1.500.000.000 in 2011 
+			initsize = capacity;
+			System.out.println("Allocating vector (ObjectArrayList) for to hold node IDs up to " + Utils.format(1L<<31));
+			chunkvector = new ObjectArrayList<short[]>();
+			chunkvector.size(1+capacity);
 		}
-		countChunkLen = new long[CHUNK_SIZE + MASK_SIZE + 1 ];
+		countChunkLen = new long[CHUNK_SIZE + MASK_SIZE + 1 ]; // used for statistics 
 		makeNewPool();
 		size = 0;
 	}
@@ -252,20 +251,39 @@
 		unassigned = arg0;
 	}
 
+	void resizeTo(int chunkid) {
+		if (chunkid <= capacity)
+			return;
+		capacity = chunkid + chunkid/8 + SIZE_INCR;
+		
+		if (capacity < 0 || capacity > 2000000000)
+			capacity = 2000000000;
+		
+		System.out.println("Resizing chunk data vector to hold IDs up to " + Utils.format((long) (1+capacity) * CHUNK_SIZE));
+		chunkvector.size(1+capacity);
+	}
+ 	
 	private short[] getChunk (int chunkid){
 		if (method == Method.hashmap)
 			return chunkmap.get(chunkid);
-		else 
-			return chunkvector[chunkid];
+		else {
+			if (chunkid <= 0 || chunkid >= chunkvector.size())
+				return null; 			
+			return chunkvector.get(chunkid);
+		}
 	}
 
 	private void putChunk (int chunkid, short[] chunk) {
 		if (method == Method.hashmap)
 			chunkmap.put(chunkid, chunk);
-		else
-			chunkvector [chunkid] = chunk;
+		else {
+			if (chunkid > capacity)
+				resizeTo(chunkid);
+			chunkvector.set(chunkid, chunk);
+		}
 	}
 
+	// store the long mask value in the 1st 4 chunk elements
 	private void storeMask (long mask, short[] chunk) {
 		// store chunkmask in chunk
 		long tmp = mask;
@@ -274,6 +292,7 @@
 			tmp = (tmp >> 16);
 		}
 	}
+	// extract the long mask value in the 1st 4 chunk elements
 	private long extractMask(short [] chunk){
 		long mask = 0;
 		for (int i = 0; i < MASK_SIZE; i++) {
@@ -290,15 +309,16 @@
 	}
 
 	@Override
-	public void stats() {
+	public void stats(int msgLevel) {
 		long usedChunks = 0;
 		long pctusage = 0;
-		for (int i=4; i <=CHUNK_SIZE + MASK_SIZE; i+=4) {
+		for (int i=8; i <=CHUNK_SIZE + MASK_SIZE; i+=4) {
 			usedChunks += countChunkLen[i];
-			if (countChunkLen[i] > 0) {
-				//	System.out.println("type-" + i + " chunks: " + Utils.format(countChunkLen[i]));
+			if (msgLevel > 0) { 
+				//System.out.println("type-" + i + " chunks: " + Utils.format(countChunkLen[i]) + " (Bytes: " + Utils.format(countChunkLen[i] * (12+i*2)));
 			}
 		}
+		
 		pctusage = Math.round((float)usedChunks*100/initsize) ;
 		if (method == Method.hashmap) { 
 			System.out.println("Chunk map details: used " + Utils.format(chunkmap.size()) + " (~ " + pctusage + "%) of the estimated " + Utils.format(initsize) + " entries." );
Index: memory_optimization/src/uk/me/parabola/splitter/SplitProcessor.java
===================================================================
--- memory_optimization/src/uk/me/parabola/splitter/SplitProcessor.java	(revision 187)
+++ memory_optimization/src/uk/me/parabola/splitter/SplitProcessor.java	(working copy)
@@ -82,10 +82,10 @@
 		}
 	}
 
-	SplitProcessor(OSMWriter[] writers, int maxThreads, long nodeCount, long maxNodeId, boolean optimizeMem) {
+	SplitProcessor(OSMWriter[] writers, int maxThreads, boolean optimizeMem) {
 		this.writers = writers;
-		this.coords = new Node2AreaMap(writers.length, nodeCount, maxNodeId, optimizeMem);
-		this.ways = new Node2AreaMap(writers.length, nodeCount/4, maxNodeId, optimizeMem); 
+		this.coords = new Node2AreaMap(writers.length, optimizeMem);
+		this.ways = new Node2AreaMap(writers.length, optimizeMem); 
 		makeWriterMap();
 		this.maxThreads = maxThreads;
 		this.toProcess = new ArrayBlockingQueue<InputQueueInfo>(writers.length); 
@@ -163,10 +163,14 @@
 
 	@Override
 	public void endMap() {
+		System.out.println("***********************************************************");
+		System.out.println("Final statistics");
+		System.out.println("***********************************************************");
 		System.out.println("coords occupancy");
-		coords.stats();
+		coords.stats(1);
 		System.out.println("ways occupancy");
-		ways.stats();
+		ways.stats(1);
+		System.out.println("");
 		for (int i = 0; i < writerInputQueues.length; i++) {
 			try {
 				writerInputQueues[i].stop();
