[mkgmap-dev] [PATCH 06/14] Binary Map parser class port to splitter.

9 Sep 2010

From: Scott Crosby <scrosby@cs.rice.edu>

---
 src/uk/me/parabola/splitter/BinaryMapParser.java |  167 ++++++++++++++++++++++
 src/uk/me/parabola/splitter/Main.java            |   56 ++++++--
 src/uk/me/parabola/splitter/MapReader.java       |   10 +-
 src/uk/me/parabola/splitter/OSMParser.java       |    6 +-
 4 files changed, 216 insertions(+), 23 deletions(-)
 create mode 100644 src/uk/me/parabola/splitter/BinaryMapParser.java

diff --git a/src/uk/me/parabola/splitter/BinaryMapParser.java b/src/uk/me/parabola/splitter/BinaryMapParser.java
new file mode 100644
index 0000000..4d76b01
--- /dev/null
+++ b/src/uk/me/parabola/splitter/BinaryMapParser.java
@@ -0,0 +1,167 @@
+package uk.me.parabola.splitter;
+
+import java.util.List;
+
+
+import crosby.binary.BinaryParser;
+import crosby.binary.Osmformat;
+
+public class BinaryMapParser extends BinaryParser {
+	// How many elements to process before displaying a status update
+	private static final int NODE_STATUS_UPDATE_THRESHOLD = 2500000;
+	private static final int WAY_STATUS_UPDATE_THRESHOLD = 500000;
+	private static final int RELATION_STATUS_UPDATE_THRESHOLD = 50000;
+
+	
+	private long nodeCount;
+	private long wayCount;
+	private long relationCount;	
+	
+	BinaryMapParser(MapProcessor processor) {
+		this.processor = processor;
+	}
+	MapProcessor processor;
+	
+		public void complete() {
+		  // End of map is sent when all input files are processed.
+		  // So do nothing.
+		}
+		
+	// Per-block state for parsing, set when processing the header of a block;
+	protected void parseDense(Osmformat.DenseNodes nodes) {
+		long last_id = 0, last_lat = 0, last_lon = 0;
+		int j = 0; 
+		for (int i=0 ; i < nodes.getIdCount(); i++) {
+			Node tmp = new Node();
+			long lat = nodes.getLat(i)+last_lat; last_lat = lat;
+			long lon = nodes.getLon(i)+last_lon; last_lon = lon;
+			long id =  nodes.getId(i)+last_id; last_id = id;
+            double latf = parseLat(lat), lonf = parseLon(lon);
+			tmp = new Node();
+			tmp.set((int)id, latf, lonf);
+            if (nodes.getKeysValsCount() > 0) {
+                while (nodes.getKeysVals(j) != 0) {
+                    int keyid = nodes.getKeysVals(j++);
+                    int valid = nodes.getKeysVals(j++);
+                    tmp.addTag(getStringById(keyid),getStringById(valid));
+                }
+                j++; // Skip over the '0' delimiter.
+            }
+			processor.processNode(tmp);
+			processNodes();
+		}
+	}
+
+	protected void parseNodes(List<Osmformat.Node> nodes) {
+		for (Osmformat.Node i : nodes) {
+			Node tmp = new Node();
+			for (int j=0 ; j < i.getKeysCount(); j++)
+				tmp.addTag(getStringById(i.getKeys(j)),getStringById(i.getVals(j)));
+			long id = i.getId();
+			double latf = parseLat(i.getLat()), lonf = parseLon(i.getLon());
+
+			tmp.set((int)id, latf, lonf);
+
+			processor.processNode(tmp);
+			processNodes();
+		}
+	}
+
+	
+	protected void parseWays(List<Osmformat.Way> ways) {
+		for (Osmformat.Way i : ways) {
+			Way tmp = new Way();
+			for (int j=0 ; j < i.getKeysCount(); j++)
+				tmp.addTag(getStringById(i.getKeys(j)),getStringById(i.getVals(j)));
+
+			long last_id=0;
+			for (long j : i.getRefsList()) {
+				tmp.addRef((int)(j+last_id));
+				last_id = j+last_id;
+			}
+
+			long id = i.getId();
+			tmp.set((int)id);
+
+			processor.processWay(tmp);
+			processWays();
+		}
+	}
+	protected void parseRelations(List<Osmformat.Relation> rels) {
+		for (Osmformat.Relation i : rels) {
+			Relation tmp = new Relation();
+			for (int j=0 ; j < i.getKeysCount(); j++)
+				tmp.addTag(getStringById(i.getKeys(j)),getStringById(i.getVals(j)));
+
+			long id = i.getId();
+			tmp.set((int)id);
+			
+			long last_mid=0;
+			for (int j =0; j < i.getMemidsCount() ; j++) {
+				long mid = last_mid + i.getMemids(j);
+				last_mid = mid;
+				String role = getStringById(i.getRolesSid(j));
+				String etype=null;
+				
+				if (i.getTypes(j) == Osmformat.Relation.MemberType.NODE)
+					etype = "node";
+				else if (i.getTypes(j) == Osmformat.Relation.MemberType.WAY)
+					etype = "way";
+				else if (i.getTypes(j) == Osmformat.Relation.MemberType.RELATION)
+					continue;
+				else
+						assert false; // TODO; Illegal file?
+
+				tmp.addMember(etype,(int)mid,role);
+			}
+			processor.processRelation(tmp);
+			processRelations();
+		}
+	}
+
+	public void parse(Osmformat.HeaderBlock block) {
+		double multiplier = .000000001;
+		double rightf = block.getBbox().getRight() * multiplier;
+		double leftf = block.getBbox().getLeft() * multiplier;
+		double topf = block.getBbox().getTop() * multiplier;
+		double bottomf = block.getBbox().getBottom() * multiplier;
+
+		for (String s : block.getRequiredFeaturesList()) {
+            if (s.equals("OsmSchema-V0.6")) continue; // OK.
+            if (s.equals("DenseNodes")) continue; // OK.
+            throw new Error("File requires unknown feature: " + s);
+        }
+
+		System.out.println("Bounding box "+leftf+" "+bottomf+" "+rightf+" "+topf);
+		
+		Area area = new Area(
+				Utils.toMapUnit(bottomf),
+				Utils.toMapUnit(leftf),
+				Utils.toMapUnit(topf),
+				Utils.toMapUnit(rightf));
+		processor.boundTag(area);
+	}
+	
+
+	private void processNodes() {
+		nodeCount++;
+		if (nodeCount % NODE_STATUS_UPDATE_THRESHOLD == 0) {
+			System.out.println(Utils.format(nodeCount) + " nodes processed...");
+		}
+
+}
+
+private void processWays()  {
+	wayCount++;
+	if (wayCount % WAY_STATUS_UPDATE_THRESHOLD == 0) {
+		System.out.println(Utils.format(wayCount) + " ways processed...");
+	}
+}
+private void processRelations()  {
+	relationCount++;
+	if (relationCount % RELATION_STATUS_UPDATE_THRESHOLD == 0) {
+		System.out.println(Utils.format(relationCount) + " ways processed...");
+	}
+}
+
+}
diff --git a/src/uk/me/parabola/splitter/Main.java b/src/uk/me/parabola/splitter/Main.java
index 84e5dd7..b5469c6 100644
--- a/src/uk/me/parabola/splitter/Main.java
+++ b/src/uk/me/parabola/splitter/Main.java
@@ -13,6 +13,9 @@
 
 package uk.me.parabola.splitter;
 
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.io.InputStreamReader;
@@ -34,6 +37,8 @@ import uk.me.parabola.splitter.geo.DummyCityFinder;
 
 import org.xmlpull.v1.XmlPullParserException;
 
+import crosby.binary.file.BlockInputStream;
+
 /**
  * Splitter for OSM files with the purpose of providing input files for mkgmap.
  * <p/>
@@ -264,14 +269,15 @@ public class Main {
 
 		MapCollector nodes = densityMap ? new DensityMapCollector(trim, resolution) : new NodeCollector();
 		MapProcessor processor = nodes;
-		MapReader mapReader = processMap(processor);
-		System.out.print("A total of " + Utils.format(mapReader.getNodeCount()) + " nodes, " +
-						Utils.format(mapReader.getWayCount()) + " ways and " +
-						Utils.format(mapReader.getRelationCount()) + " relations were processed ");
+		//MapReader mapReader = 
+		processMap(processor);
+		//System.out.print("A total of " + Utils.format(mapReader.getNodeCount()) + " nodes, " +
+		//				Utils.format(mapReader.getWayCount()) + " ways and " +
+		//				Utils.format(mapReader.getRelationCount()) + " relations were processed ");
 
-		System.out.println("in " + filenames.size() + (filenames.size() == 1 ? " file" : " files"));
-		System.out.println("Min node ID = " + mapReader.getMinNodeId());
-		System.out.println("Max node ID = " + mapReader.getMaxNodeId());
+		//System.out.println("in " + filenames.size() + (filenames.size() == 1 ? " file" : " files"));
+		//System.out.println("Min node ID = " + mapReader.getMinNodeId());
+		//System.out.println("Max node ID = " + mapReader.getMaxNodeId());
 
 		System.out.println("Time: " + new Date());
 
@@ -342,10 +348,10 @@ public class Main {
 							areas.get(i * areasPerPass + currentWriters.length - 1).getMapId() + ')');
 
 			MapProcessor processor = new SplitProcessor(currentWriters, maxThreads);
-			MapReader mapReader = processMap(processor);
-			System.out.println("Wrote " + Utils.format(mapReader.getNodeCount()) + " nodes, " +
-							Utils.format(mapReader.getWayCount()) + " ways, " +
-							Utils.format(mapReader.getRelationCount()) + " relations");
+			processMap(processor);
+			//System.out.println("Wrote " + Utils.format(mapReader.getNodeCount()) + " nodes, " +
+			//				Utils.format(mapReader.getWayCount()) + " ways, " +
+			//				Utils.format(mapReader.getRelationCount()) + " relations");
 		}
 	}
 
@@ -366,12 +372,32 @@ public class Main {
 			parse(parser, reader);
 		} else {
 			for (String filename : filenames) {
-				System.out.println("Processing " + filename + "...");
-				Reader reader = Utils.openFile(filename, maxThreads > 1);
-				parse(parser, reader);
+				try {
+					if (filename.endsWith(".bin")) {
+						// Is it a binary file?
+						File file = new File(filename);
+						BlockInputStream blockinput = (new BlockInputStream(
+										       new FileInputStream(file), new BinaryMapParser(processor)));
+						try {
+							blockinput.process();
+						} finally {
+							blockinput.close();
+						}
+					} else {
+						// No, try XML.
+						Reader reader = Utils.openFile(filename, maxThreads > 1);
+						parse(parser, reader);
+					}
+				} catch (FileNotFoundException e) {
+					e.printStackTrace();
+				} catch (XmlPullParserException e) {
+					e.printStackTrace();
+				} catch (IOException e) {
+					e.printStackTrace();
+				}
 			}
 		}
-		parser.endMap();
+		processor.endMap();
 	}
 
 	private void parse(OSMParser parser, Reader reader) throws IOException, XmlPullParserException {
diff --git a/src/uk/me/parabola/splitter/MapReader.java b/src/uk/me/parabola/splitter/MapReader.java
index 8afa12b..d8a8a21 100644
--- a/src/uk/me/parabola/splitter/MapReader.java
+++ b/src/uk/me/parabola/splitter/MapReader.java
@@ -14,13 +14,13 @@
 package uk.me.parabola.splitter;
 
 public interface MapReader {
-	long getNodeCount();
+	//long getNodeCount();
 
-	long getWayCount();
+	//long getWayCount();
 
-	long getRelationCount();
+	//long getRelationCount();
 
-	int getMinNodeId();
+	//int getMinNodeId();
 
-	int getMaxNodeId();
+	//int getMaxNodeId();
 }
diff --git a/src/uk/me/parabola/splitter/OSMParser.java b/src/uk/me/parabola/splitter/OSMParser.java
index 50569be..1388261 100644
--- a/src/uk/me/parabola/splitter/OSMParser.java
+++ b/src/uk/me/parabola/splitter/OSMParser.java
@@ -51,7 +51,7 @@ class OSMParser extends AbstractXppParser implements MapReader {
 		this.startNodeOnly = processor.isStartNodeOnly();
 		this.mixed = mixed;
 	}
-
+	/*
 	@Override
 	public long getNodeCount() {
 		return nodeCount;
@@ -76,11 +76,11 @@ class OSMParser extends AbstractXppParser implements MapReader {
 	public int getMaxNodeId() {
 		return maxNodeId;
 	}
-
+	*/
 	public void endMap() {
 		processor.endMap();
 	}
-
+	
 	/**
 	 * Receive notification of the start of an element.
 	 */
-- 
1.7.2.3

    

[mkgmap-dev] [PATCH 06/14] Binary Map parser class port to splitter.

Jeffrey C. Ollie