();
- private boolean wantHelp;
- private int maxParamLength;
-
- public P parse(Class
paramInterface, String... args) {
- if (!paramInterface.isInterface()) {
- throw new IllegalArgumentException(paramInterface + " must be an interface");
- }
- return createProxy(paramInterface, args);
- }
-
- public Map getValidParams() {
- return paramMap;
- }
-
- public Map getConvertedParams() {
- return convertedParamMap;
- }
-
- public List getAdditionalParams() {
- return additionalParams;
- }
-
- public List getErrors() {
- return errors;
- }
-
- private P createProxy(Class
paramInterface, String... args) {
-
- Map params = new HashMap();
- paramMap.clear();
- convertedParamMap.clear();
- wantHelp = false;
- for (Method method : paramInterface.getDeclaredMethods()) {
- Option option = ReflectionUtils.getOptionAnnotation(method);
- String name = getParameterName(method, option);
- if (name.length() > maxParamLength) {
- maxParamLength = name.length();
- }
- String description = option.description();
- String defaultValue = option.defaultValue();
- if (defaultValue.equals(Option.OPTIONAL)) {
- defaultValue = null;
- }
- Class returnType = ReflectionUtils.getBoxedClass(method.getReturnType());
- Param param = new Param(name, description, defaultValue, returnType);
- paramMap.put(name, param);
- MethodParamPair pair = new MethodParamPair(method, param);
- params.put(name, pair);
- }
-
- Map valuesMap = convert(params, args);
- for (Map.Entry entry : valuesMap.entrySet()) {
- Method method = entry.getKey();
- Option option = ReflectionUtils.getOptionAnnotation(method);
- String name = getParameterName(method, option);
- convertedParamMap.put(name, entry.getValue());
- }
-
- if (wantHelp) {
- displayUsage();
- System.exit(0);
- }
- ParamInvocationHandler invocationHandler = new ParamInvocationHandler(valuesMap);
- return (P) Proxy.newProxyInstance(paramInterface.getClassLoader(), new Class[]{paramInterface}, invocationHandler);
- }
-
- private Map convert(Map paramMap, String[] args) {
- Map result = new HashMap(10);
-
- // First set up the defaults
- for (MethodParamPair pair : paramMap.values()) {
- Method method = pair.getMethod();
- Param param = pair.getParam();
- Object value = converter.convert(param, param.getDefaultValue());
- if (value == null && method.getReturnType().isPrimitive()) {
- result.put(method, converter.getPrimitiveDefault(method.getReturnType()));
- } else {
- result.put(method, value);
- }
- }
-
- // Now override these with any parameters that were specified on the command line
- for (String arg : args) {
- if (arg.startsWith("--")) {
- String name;
- String value;
- int j = arg.indexOf('=');
- if (j > 0) {
- name = arg.substring(2, j);
- value = arg.substring(j + 1);
- } else {
- // Should be a boolean
- name = arg.substring(2);
- value = null;
- }
- MethodParamPair pair = paramMap.get(name);
- if (pair != null) {
- if (pair.getParam().getReturnType() == Boolean.class && value == null) {
- result.put(pair.getMethod(), Boolean.TRUE);
- } else {
- try {
- Object convertedVal = converter.convert(pair.getParam(), value);
- result.put(pair.getMethod(), convertedVal);
- } catch (Exception e) {
- errors.add("Unable to parse " + arg + ". Reason: " + e.getMessage());
- }
- }
- } else {
- // Unknown parameter
- if ("help".equals(name)) {
- wantHelp = true;
- } else {
- errors.add("Parameter " + arg + " is not recognised, ignoring");
- }
- }
- } else {
- // We have a parameter that doesn't start with --
- additionalParams.add(arg);
- }
- }
- return result;
- }
-
- public void displayUsage() {
- System.out.println("Usage:");
- StringBuilder buf = new StringBuilder(100);
- for (Param param : paramMap.values()) {
- buf.setLength(0);
- buf.append(" --").append(param.getName());
- for (int i = 0; i < maxParamLength - param.getName().length() + 2; i++) {
- buf.append(' ');
- }
- buf.append(param.getDescription());
- if (param.getDefaultValue() != null) {
- buf.append(" Default is ").append(param.getDefaultValue()).append('.');
- }
- System.out.println(buf.toString());
- }
- }
-
- private
String getParameterName(Method method, Option option) {
- return option.name().length() == 0 ? ReflectionUtils.getParamName(method) : option.name();
- }
-
- private static class ParamInvocationHandler implements InvocationHandler {
- private final Map valuesMap;
-
- private ParamInvocationHandler(Map valuesMap) {
- this.valuesMap = valuesMap;
- }
-
- @Override
- public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
- return valuesMap.get(method);
- }
- }
-
- private static class MethodParamPair {
- private final Method method;
- private final Param param;
-
- private MethodParamPair(Method method, Param param) {
- this.method = method;
- this.param = param;
- }
-
- public Method getMethod() {
- return method;
- }
-
- public Param getParam() {
- return param;
- }
- }
+/*
+ * Copyright (c) 2009.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 3 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ */
+
+package uk.me.parabola.splitter.args;
+
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+/**
+ * Parses command line arguments and returns them via the supplied interface.
+ *
+ * @author Chris Miller
+ */
+public class ParamParser {
+
+ private final ParamConverter converter = new ParamConverter();
+ private final Map paramMap = new TreeMap();
+ private final Map convertedParamMap = new TreeMap();
+ private final List additionalParams = new ArrayList();
+ private final List errors = new ArrayList();
+ private boolean wantHelp;
+ private int maxParamLength;
+
+ public P parse(Class
paramInterface, String... args) {
+ if (!paramInterface.isInterface()) {
+ throw new IllegalArgumentException(paramInterface + " must be an interface");
+ }
+ return createProxy(paramInterface, args);
+ }
+
+ public Map getValidParams() {
+ return paramMap;
+ }
+
+ public Map getConvertedParams() {
+ return convertedParamMap;
+ }
+
+ public List getAdditionalParams() {
+ return additionalParams;
+ }
+
+ public List getErrors() {
+ return errors;
+ }
+
+ private P createProxy(Class
paramInterface, String... args) {
+
+ Map params = new HashMap();
+ paramMap.clear();
+ convertedParamMap.clear();
+ wantHelp = false;
+ for (Method method : paramInterface.getDeclaredMethods()) {
+ Option option = ReflectionUtils.getOptionAnnotation(method);
+ String name = getParameterName(method, option);
+ if (name.length() > maxParamLength) {
+ maxParamLength = name.length();
+ }
+ String description = option.description();
+ String defaultValue = option.defaultValue();
+ if (defaultValue.equals(Option.OPTIONAL)) {
+ defaultValue = null;
+ }
+ Class returnType = ReflectionUtils.getBoxedClass(method.getReturnType());
+ Param param = new Param(name, description, defaultValue, returnType);
+ paramMap.put(name, param);
+ MethodParamPair pair = new MethodParamPair(method, param);
+ params.put(name, pair);
+ }
+
+ Map valuesMap = convert(params, args);
+ for (Map.Entry entry : valuesMap.entrySet()) {
+ Method method = entry.getKey();
+ Option option = ReflectionUtils.getOptionAnnotation(method);
+ String name = getParameterName(method, option);
+ convertedParamMap.put(name, entry.getValue());
+ }
+
+ if (wantHelp) {
+ displayUsage();
+ System.exit(0);
+ }
+ ParamInvocationHandler invocationHandler = new ParamInvocationHandler(valuesMap);
+ return (P) Proxy.newProxyInstance(paramInterface.getClassLoader(), new Class[]{paramInterface}, invocationHandler);
+ }
+
+ private Map convert(Map paramMap, String[] args) {
+ Map result = new HashMap(10);
+
+ // First set up the defaults
+ for (MethodParamPair pair : paramMap.values()) {
+ Method method = pair.getMethod();
+ Param param = pair.getParam();
+ Object value = converter.convert(param, param.getDefaultValue());
+ if (value == null && method.getReturnType().isPrimitive()) {
+ result.put(method, converter.getPrimitiveDefault(method.getReturnType()));
+ } else {
+ result.put(method, value);
+ }
+ }
+
+ // Now override these with any parameters that were specified on the command line
+ for (String arg : args) {
+ if (arg.startsWith("--")) {
+ String name;
+ String value;
+ int j = arg.indexOf('=');
+ if (j > 0) {
+ name = arg.substring(2, j);
+ value = arg.substring(j + 1);
+ } else {
+ // Should be a boolean
+ name = arg.substring(2);
+ value = null;
+ }
+ MethodParamPair pair = paramMap.get(name);
+ if (pair != null) {
+ if (pair.getParam().getReturnType() == Boolean.class && value == null) {
+ result.put(pair.getMethod(), Boolean.TRUE);
+ } else {
+ try {
+ Object convertedVal = converter.convert(pair.getParam(), value);
+ result.put(pair.getMethod(), convertedVal);
+ } catch (Exception e) {
+ errors.add("Unable to parse " + arg + ". Reason: " + e.getMessage());
+ }
+ }
+ } else {
+ // Unknown parameter
+ if ("help".equals(name)) {
+ wantHelp = true;
+ } else {
+ errors.add("Parameter " + arg + " is not recognised, ignoring");
+ }
+ }
+ } else {
+ // We have a parameter that doesn't start with --
+ additionalParams.add(arg);
+ }
+ }
+ return result;
+ }
+
+ public void displayUsage() {
+ System.out.println("Usage:");
+ StringBuilder buf = new StringBuilder(100);
+ for (Param param : paramMap.values()) {
+ buf.setLength(0);
+ buf.append(" --").append(param.getName());
+ for (int i = 0; i < maxParamLength - param.getName().length() + 2; i++) {
+ buf.append(' ');
+ }
+ buf.append(param.getDescription());
+ if (param.getDefaultValue() != null) {
+ buf.append(" Default is ").append(param.getDefaultValue()).append('.');
+ }
+ System.out.println(buf.toString());
+ }
+ }
+
+ private
String getParameterName(Method method, Option option) {
+ return option.name().length() == 0 ? ReflectionUtils.getParamName(method) : option.name();
+ }
+
+ private static class ParamInvocationHandler implements InvocationHandler {
+ private final Map valuesMap;
+
+ private ParamInvocationHandler(Map valuesMap) {
+ this.valuesMap = valuesMap;
+ }
+
+ public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
+ return valuesMap.get(method);
+ }
+ }
+
+ private static class MethodParamPair {
+ private final Method method;
+ private final Param param;
+
+ private MethodParamPair(Method method, Param param) {
+ this.method = method;
+ this.param = param;
+ }
+
+ public Method getMethod() {
+ return method;
+ }
+
+ public Param getParam() {
+ return param;
+ }
+ }
}
\ No newline at end of file
Index: src/uk/me/parabola/splitter/BinaryMapWriter.java
===================================================================
--- src/uk/me/parabola/splitter/BinaryMapWriter.java (revision 0)
+++ src/uk/me/parabola/splitter/BinaryMapWriter.java (revision 0)
@@ -0,0 +1,528 @@
+/*
+ * Copyright (c) 2009.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 3 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ */
+
+package uk.me.parabola.splitter;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Formatter;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+
+import uk.me.parabola.splitter.Relation.Member;
+import crosby.binary.BinarySerializer;
+import crosby.binary.Osmformat;
+import crosby.binary.StringTable;
+import crosby.binary.Osmformat.DenseInfo;
+import crosby.binary.Osmformat.Relation.MemberType;
+import crosby.binary.file.BlockOutputStream;
+import crosby.binary.file.FileBlock;
+
+public class BinaryMapWriter extends OSMWriter {
+
+ private PBFSerializer serializer;
+
+ private BlockOutputStream output;
+
+ protected boolean useDense = true;
+
+ protected boolean headerWritten = false;
+
+ public class PBFSerializer extends BinarySerializer {
+
+ public PBFSerializer(BlockOutputStream output)
+ {
+ super(output);
+ omit_metadata = true;
+ }
+
+ /** Base class containing common code needed for serializing each type of primitives. */
+ private abstract class Prim {
+ /** Queue that tracks the list of all primitives. */
+ ArrayList contents = new ArrayList();
+
+ /** Add to the queue.
+ * @param item The entity to add */
+ public void add(T item)
+ {
+ contents.add(item);
+ }
+
+ /** Add all of the tags of all entities in the queue to the stringtable. */
+ public void addStringsToStringtable()
+ {
+ StringTable stable = getStringTable();
+ for(T i : contents) {
+ Iterator tags = i.tagsIterator();
+ while(tags.hasNext()) {
+ Element.Tag tag = tags.next();
+ stable.incr(tag.getKey());
+ stable.incr(tag.getValue());
+ }
+ if(!omit_metadata) {
+ // stable.incr(i.getUser().getName());
+ }
+ }
+ }
+
+ // private static final int MAXWARN = 100;
+
+ public void serializeMetadataDense(DenseInfo.Builder b,
+ List entities)
+ {
+ if(omit_metadata) {
+ return;
+ }
+
+ // long lasttimestamp = 0, lastchangeset = 0;
+ // int lastuserSid = 0, lastuid = 0;
+ // StringTable stable = serializer.getStringTable();
+ // for(Element e : entities) {
+ //
+ // if(e.getUser() == OsmUser.NONE && warncount < MAXWARN) {
+ // LOG
+ // .warning("Attention: Data being output lacks metadata. Please use omitmetadata=true");
+ // warncount++;
+ // }
+ // int uid = e.getUser().getId();
+ // int userSid = stable.getIndex(e.getUser().getName());
+ // int timestamp = (int)(e.getTimestamp().getTime() / date_granularity);
+ // int version = e.getVersion();
+ // long changeset = e.getChangesetId();
+ //
+ // b.addVersion(version);
+ // b.addTimestamp(timestamp - lasttimestamp);
+ // lasttimestamp = timestamp;
+ // b.addChangeset(changeset - lastchangeset);
+ // lastchangeset = changeset;
+ // b.addUid(uid - lastuid);
+ // lastuid = uid;
+ // b.addUserSid(userSid - lastuserSid);
+ // lastuserSid = userSid;
+ // }
+ }
+
+ public Osmformat.Info.Builder serializeMetadata(Element e)
+ {
+ // StringTable stable = serializer.getStringTable();
+ Osmformat.Info.Builder b = Osmformat.Info.newBuilder();
+ if(!omit_metadata) {
+ // if(e.getUser() == OsmUser.NONE && warncount < MAXWARN) {
+ // LOG
+ // .warning("Attention: Data being output lacks metadata. Please use omitmetadata=true");
+ // warncount++;
+ // }
+ // if(e.getUser() != OsmUser.NONE) {
+ // b.setUid(e.getUser().getId());
+ // b.setUserSid(stable.getIndex(e.getUser().getName()));
+ // }
+ // b.setTimestamp((int)(e.getTimestamp().getTime() / date_granularity));
+ // b.setVersion(e.getVersion());
+ // b.setChangeset(e.getChangesetId());
+ }
+ return b;
+ }
+ }
+
+ private class NodeGroup extends Prim implements
+ PrimGroupWriterInterface {
+
+ public Osmformat.PrimitiveGroup serialize()
+ {
+ if(useDense) {
+ return serializeDense();
+ } else {
+ return serializeNonDense();
+ }
+ }
+
+ /**
+ * Serialize all nodes in the 'dense' format.
+ */
+ public Osmformat.PrimitiveGroup serializeDense()
+ {
+ if(contents.size() == 0) {
+ return null;
+ }
+ // System.out.format("%d Dense ",nodes.size());
+ Osmformat.PrimitiveGroup.Builder builder = Osmformat.PrimitiveGroup
+ .newBuilder();
+ StringTable stable = serializer.getStringTable();
+
+ long lastlat = 0, lastlon = 0, lastid = 0;
+ Osmformat.DenseNodes.Builder bi = Osmformat.DenseNodes.newBuilder();
+ boolean doesBlockHaveTags = false;
+ // Does anything in this block have tags?
+ for(Node i : contents) {
+ doesBlockHaveTags = doesBlockHaveTags || (i.tagsIterator().hasNext());
+ }
+ if(!omit_metadata) {
+ Osmformat.DenseInfo.Builder bdi = Osmformat.DenseInfo.newBuilder();
+ serializeMetadataDense(bdi, contents);
+ bi.setDenseinfo(bdi);
+ }
+
+ for(Node i : contents) {
+ long id = i.getId();
+ int lat = mapDegrees(i.getLat());
+ int lon = mapDegrees(i.getLon());
+ bi.addId(id - lastid);
+ lastid = id;
+ bi.addLon(lon - lastlon);
+ lastlon = lon;
+ bi.addLat(lat - lastlat);
+ lastlat = lat;
+
+ // Then we must include tag information.
+ if(doesBlockHaveTags) {
+ Iterator tags = i.tagsIterator();
+ while(tags.hasNext()) {
+ Element.Tag t = tags.next();
+ bi.addKeysVals(stable.getIndex(t.getKey()));
+ bi.addKeysVals(stable.getIndex(t.getValue()));
+ }
+ bi.addKeysVals(0); // Add delimiter.
+ }
+ }
+ builder.setDense(bi);
+ return builder.build();
+ }
+
+ /**
+ * Serialize all nodes in the non-dense format.
+ *
+ * @param parentbuilder Add to this PrimitiveBlock.
+ */
+ public Osmformat.PrimitiveGroup serializeNonDense()
+ {
+ if(contents.size() == 0) {
+ return null;
+ }
+ // System.out.format("%d Nodes ",nodes.size());
+ StringTable stable = serializer.getStringTable();
+ Osmformat.PrimitiveGroup.Builder builder = Osmformat.PrimitiveGroup
+ .newBuilder();
+ for(Node i : contents) {
+ long id = i.getId();
+ int lat = mapDegrees(i.getLat());
+ int lon = mapDegrees(i.getLon());
+ Osmformat.Node.Builder bi = Osmformat.Node.newBuilder();
+ bi.setId(id);
+ bi.setLon(lon);
+ bi.setLat(lat);
+ Iterator tags = i.tagsIterator();
+ while(tags.hasNext()) {
+ Element.Tag t = tags.next();
+ bi.addKeys(stable.getIndex(t.getKey()));
+ bi.addVals(stable.getIndex(t.getValue()));
+ }
+ if(!omit_metadata) {
+ bi.setInfo(serializeMetadata(i));
+ }
+ builder.addNodes(bi);
+ }
+ return builder.build();
+ }
+
+ }
+
+ private class WayGroup extends Prim implements
+ PrimGroupWriterInterface {
+ public Osmformat.PrimitiveGroup serialize()
+ {
+ if(contents.size() == 0) {
+ return null;
+ }
+
+ // System.out.format("%d Ways ",contents.size());
+ StringTable stable = serializer.getStringTable();
+ Osmformat.PrimitiveGroup.Builder builder = Osmformat.PrimitiveGroup
+ .newBuilder();
+ for(Way i : contents) {
+ Osmformat.Way.Builder bi = Osmformat.Way.newBuilder();
+ bi.setId(i.getId());
+ long lastid = 0;
+ for(int j : i.getRefs()) {
+ long id = j;
+ bi.addRefs(id - lastid);
+ lastid = id;
+ }
+ Iterator tags = i.tagsIterator();
+ while(tags.hasNext()) {
+ Element.Tag t = tags.next();
+ bi.addKeys(stable.getIndex(t.getKey()));
+ bi.addVals(stable.getIndex(t.getValue()));
+ }
+ if(!omit_metadata) {
+ bi.setInfo(serializeMetadata(i));
+ }
+ builder.addWays(bi);
+ }
+ return builder.build();
+ }
+ }
+
+ private class RelationGroup extends Prim implements
+ PrimGroupWriterInterface {
+ public void addStringsToStringtable()
+ {
+ StringTable stable = serializer.getStringTable();
+ super.addStringsToStringtable();
+ for(Relation i : contents) {
+ for(Member j : i.getMembers()) {
+ stable.incr(j.getRole());
+ }
+ }
+ }
+
+ public Osmformat.PrimitiveGroup serialize()
+ {
+ if(contents.size() == 0) {
+ return null;
+ }
+
+ // System.out.format("%d Relations ",contents.size());
+ StringTable stable = serializer.getStringTable();
+ Osmformat.PrimitiveGroup.Builder builder = Osmformat.PrimitiveGroup
+ .newBuilder();
+ for(Relation i : contents) {
+ Osmformat.Relation.Builder bi = Osmformat.Relation.newBuilder();
+ bi.setId(i.getId());
+ Member[] arr = new Member[i.getMembers().size()];
+ i.getMembers().toArray(arr);
+ long lastid = 0;
+ for(Member j : i.getMembers()) {
+ long id = j.getRef();
+ bi.addMemids(id - lastid);
+ lastid = id;
+ if(j.getType().equals("node")) {
+ bi.addTypes(MemberType.NODE);
+ } else if(j.getType().equals("way")) {
+ bi.addTypes(MemberType.WAY);
+ } else if(j.getType().equals("relation")) {
+ bi.addTypes(MemberType.RELATION);
+ } else {
+ assert (false); // Software bug: Unknown entity.
+ }
+ bi.addRolesSid(stable.getIndex(j.getRole()));
+ }
+
+ Iterator tags = i.tagsIterator();
+ while(tags.hasNext()) {
+ Element.Tag t = tags.next();
+ bi.addKeys(stable.getIndex(t.getKey()));
+ bi.addVals(stable.getIndex(t.getValue()));
+ }
+ if(!omit_metadata) {
+ bi.setInfo(serializeMetadata(i));
+ }
+ builder.addRelations(bi);
+ }
+ return builder.build();
+ }
+ }
+
+ /* One list for each type */
+ private WayGroup ways;
+
+ private NodeGroup nodes;
+
+ private RelationGroup relations;
+
+ private Processor processor = new Processor();
+
+ /**
+ * Buffer up events into groups that are all of the same type, or all of the
+ * same length, then process each buffer.
+ */
+ public class Processor {
+
+ /**
+ * Check if we've reached the batch size limit and process the batch if
+ * we have.
+ */
+ public void checkLimit()
+ {
+ total_entities++;
+ if(++batch_size < batch_limit) {
+ return;
+ }
+ switchTypes();
+ processBatch();
+ }
+
+ public void process(Node node)
+ {
+ if(nodes == null) {
+ writeEmptyHeaderIfNeeded();
+ // Need to switch types.
+ switchTypes();
+ nodes = new NodeGroup();
+ }
+ nodes.add(node);
+ checkLimit();
+ }
+
+ public void process(Way way)
+ {
+ if(ways == null) {
+ writeEmptyHeaderIfNeeded();
+ switchTypes();
+ ways = new WayGroup();
+ }
+ ways.add(way);
+ checkLimit();
+ }
+
+ public void process(Relation relation)
+ {
+ if(relations == null) {
+ writeEmptyHeaderIfNeeded();
+ switchTypes();
+ relations = new RelationGroup();
+ }
+ relations.add(relation);
+ checkLimit();
+ }
+ }
+
+ /**
+ * At the end of this function, all of the lists of unprocessed 'things'
+ * must be null
+ */
+ private void switchTypes()
+ {
+ if(nodes != null) {
+ groups.add(nodes);
+ nodes = null;
+ } else if(ways != null) {
+ groups.add(ways);
+ ways = null;
+ } else if(relations != null) {
+ groups.add(relations);
+ relations = null;
+ } else {
+ return; // No data. Is this an empty file?
+ }
+ }
+
+ /** Write empty header block when there's no bounds entity. */
+ public void writeEmptyHeaderIfNeeded()
+ {
+ if(headerWritten) {
+ return;
+ }
+ Osmformat.HeaderBlock.Builder headerblock = Osmformat.HeaderBlock
+ .newBuilder();
+ finishHeader(headerblock);
+ }
+ }
+
+ public BinaryMapWriter(Area bounds, File outputDir)
+ {
+ super(bounds, outputDir);
+ }
+
+ public void initForWrite(int mapId, int extra)
+ {
+ extendedBounds = new Area(bounds.getMinLat() - extra, bounds.getMinLong()
+ - extra, bounds.getMaxLat() + extra, bounds.getMaxLong() + extra);
+
+ String filename = new Formatter()
+ .format(Locale.ROOT, "%08d.osm.pbf", mapId).toString();
+ try {
+ output = new BlockOutputStream(new FileOutputStream(new File(outputDir,
+ filename)));
+ serializer = new PBFSerializer(output);
+ writeHeader();
+ }
+ catch(IOException e) {
+ System.out.println("Could not open or write file header. Reason: "
+ + e.getMessage());
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void writeHeader() throws IOException
+ {
+ Osmformat.HeaderBlock.Builder headerblock = Osmformat.HeaderBlock
+ .newBuilder();
+
+ Osmformat.HeaderBBox.Builder bbox = Osmformat.HeaderBBox.newBuilder();
+ System.out.println(bounds);
+ bbox.setLeft(serializer.mapRawDegrees(Utils.toDegrees(bounds.getMinLong())));
+ bbox.setBottom(serializer.mapRawDegrees(Utils.toDegrees(bounds.getMinLat())));
+ bbox.setRight(serializer.mapRawDegrees(Utils.toDegrees(bounds.getMaxLong())));
+ bbox.setTop(serializer.mapRawDegrees(Utils.toDegrees(bounds.getMaxLat())));
+ headerblock.setBbox(bbox);
+
+ // headerblock.setSource("splitter"); //TODO: entity.getOrigin());
+ finishHeader(headerblock);
+ }
+
+ /** Write the header fields that are always needed.
+ *
+ * @param headerblock Incomplete builder to complete and write.
+ * */
+ public void finishHeader(Osmformat.HeaderBlock.Builder headerblock)
+ {
+ headerblock.setWritingprogram("splitter-r171");
+ headerblock.addRequiredFeatures("OsmSchema-V0.6");
+ if(useDense) {
+ headerblock.addRequiredFeatures("DenseNodes");
+ }
+ Osmformat.HeaderBlock message = headerblock.build();
+ try {
+ output.write(FileBlock.newInstance("OSMHeader", message.toByteString(),
+ null));
+ }
+ catch(IOException e) {
+ throw new RuntimeException("Unable to write OSM header.", e);
+ }
+ headerWritten = true;
+ }
+
+ public void finishWrite()
+ {
+ try {
+ serializer.processBatch();
+ serializer.close();
+ }
+ catch(IOException e) {
+ System.out.println("Could not write end of file: " + e);
+ }
+ }
+
+ public boolean nodeBelongsToThisArea(Node node)
+ {
+ return (extendedBounds.contains(node.getMapLat(), node.getMapLon()));
+ }
+
+ public void write(Node node)
+ {
+ serializer.processor.process(node);
+ }
+
+ public void write(Way way)
+ {
+ serializer.processor.process(way);
+ }
+
+ public void write(Relation relation)
+ {
+ serializer.processor.process(relation);
+ }
+}
Index: src/uk/me/parabola/splitter/MapCollector.java
===================================================================
--- src/uk/me/parabola/splitter/MapCollector.java (revision 171)
+++ src/uk/me/parabola/splitter/MapCollector.java (working copy)
@@ -1,20 +1,22 @@
-/*
- * Copyright (c) 2009.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 3 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * General Public License for more details.
- */
-
-package uk.me.parabola.splitter;
-
-public interface MapCollector extends MapProcessor {
- Area getExactArea();
-
- SplittableArea getRoundedArea(int resolution);
-}
+/*
+ * Copyright (c) 2009.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 3 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ */
+
+package uk.me.parabola.splitter;
+
+public interface MapCollector extends MapProcessor {
+ Area getExactArea();
+
+ SplittableArea getRoundedArea(int resolution);
+
+ void boundTag(Area bounds);
+}
Index: src/uk/me/parabola/splitter/Main.java
===================================================================
--- src/uk/me/parabola/splitter/Main.java (revision 171)
+++ src/uk/me/parabola/splitter/Main.java (working copy)
@@ -1,455 +1,462 @@
-/*
- * Copyright (c) 2009.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 3 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * General Public License for more details.
- */
-
-package uk.me.parabola.splitter;
-
-import crosby.binary.file.BlockInputStream;
-import org.xmlpull.v1.XmlPullParserException;
-import uk.me.parabola.splitter.args.ParamParser;
-import uk.me.parabola.splitter.args.SplitterParams;
-import uk.me.parabola.splitter.geo.City;
-import uk.me.parabola.splitter.geo.CityFinder;
-import uk.me.parabola.splitter.geo.CityLoader;
-import uk.me.parabola.splitter.geo.DefaultCityFinder;
-import uk.me.parabola.splitter.geo.DummyCityFinder;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.PrintWriter;
-import java.io.Reader;
-import java.nio.charset.Charset;
-import java.util.Date;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * Splitter for OSM files with the purpose of providing input files for mkgmap.
- *
- * The input file is split so that no piece has more than a given number of nodes in it.
- *
- * @author Steve Ratcliffe
- */
-public class Main {
- private static final String DEFAULT_DIR = ".";
-
- // We can only process a maximum of 255 areas at a time because we
- // compress an area ID into 8 bits to save memory (and 0 is reserved)
- private int maxAreasPerPass;
-
- // A list of the OSM files to parse.
- private List filenames;
-
- // The description to write into the template.args file.
- private String description;
-
- // The starting map ID.
- private int mapId;
-
- // The amount in map units that tiles overlap (note that the final img's will not overlap
- // but the input files do).
- private int overlapAmount;
-
- // The max number of nodes that will appear in a single file.
- private int maxNodes;
-
- // The maximum resolution of the map to be produced by mkgmap. This is a value in the range
- // 0-24. Higher numbers mean higher detail. The resolution determines how the tiles must
- // be aligned. Eg a resolution of 13 means the tiles need to have their edges aligned to
- // multiples of 2 ^ (24 - 13) = 2048 map units, and their widths and heights must be a multiple
- // of 2 * 2 ^ (24 - 13) = 4096 units. The tile widths and height multiples are double the tile
- // alignment because the center point of the tile is stored, and that must be aligned the
- // same as the tile edges are.
- private int resolution;
-
- // Whether or not to trim tiles of any empty space around their edges.
- private boolean trim;
- // This gets set if no osm file is supplied as a parameter and the cache is empty.
- private boolean useStdIn;
- // Set if there is a previous area file given on the command line.
- private AreaList areaList;
- // Whether or not the source OSM file(s) contain strictly nodes first, then ways, then rels,
- // or they're all mixed up. Running with mixed enabled takes longer.
- private boolean mixed;
- // The path where the results are written out to.
- private File fileOutputDir;
- // A GeoNames file to use for naming the tiles.
- private String geoNamesFile;
- // How often (in seconds) to provide JVM status information. Zero = no information.
- private int statusFreq;
- // Whether to use the density map. Disabling this (not recommended) causes the splitter to
- // revert to using legacy mode which takes MUCH more memory during phase one.
- private boolean densityMap;
-
- private String kmlOutputFile;
- // The maximum number of threads the splitter should use.
- private int maxThreads;
-
- public static void main(String[] args) {
-
- Main m = new Main();
- m.start(args);
- }
-
- private void start(String[] args) {
- readArgs(args);
- if (statusFreq > 0) {
- JVMHealthMonitor.start(statusFreq);
- }
- long start = System.currentTimeMillis();
- System.out.println("Time started: " + new Date());
- try {
- split();
- } catch (IOException e) {
- System.err.println("Error opening or reading file " + e);
- e.printStackTrace();
- } catch (XmlPullParserException e) {
- System.err.println("Error parsing xml from file " + e);
- e.printStackTrace();
- }
- System.out.println("Time finished: " + new Date());
- System.out.println("Total time taken: " + (System.currentTimeMillis() - start) / 1000 + 's');
- }
-
- private void split() throws IOException, XmlPullParserException {
-
- File outputDir = fileOutputDir;
- if (!outputDir.exists()) {
- System.out.println("Output directory not found. Creating directory '" + fileOutputDir + "'");
- if (!outputDir.mkdirs()) {
- System.err.println("Unable to create output directory! Using default directory instead");
- fileOutputDir = new File(DEFAULT_DIR);
- }
- } else if (!outputDir.isDirectory()) {
- System.err.println("The --output-dir parameter must specify a directory. The --output-dir parameter is being ignored, writing to default directory instead.");
- fileOutputDir = new File(DEFAULT_DIR);
- }
-
- if (filenames.isEmpty()) {
- if (areaList == null) {
- throw new IllegalArgumentException("No .osm files were supplied so at least one of --cache or --split-file must be specified");
- } else {
- int areaCount = areaList.getAreas().size();
- int passes = getAreasPerPass(areaCount);
- if (passes > 1) {
- throw new IllegalArgumentException("No .osm files or --cache parameter were supplied, but stdin cannot be used because " + passes
- + " passes are required to write out the areas. Either provide --cache or increase --max-areas to match the number of areas (" + areaCount + ')');
- }
- useStdIn = true;
- }
- }
-
- if (areaList == null) {
- int alignment = 1 << (24 - resolution);
- System.out.println("Map is being split for resolution " + resolution + ':');
- System.out.println(" - area boundaries are aligned to 0x" + Integer.toHexString(alignment) + " map units");
- System.out.println(" - areas are multiples of 0x" + Integer.toHexString(alignment * 2) + " map units wide and high");
- areaList = calculateAreas();
- for (Area area : areaList.getAreas()) {
- area.setMapId(mapId++);
- }
- nameAreas();
- areaList.write(new File(fileOutputDir, "areas.list").getPath());
- } else {
- nameAreas();
- }
-
- List areas = areaList.getAreas();
- System.out.println(areas.size() + " areas:");
- for (Area area : areas) {
- System.out.print("Area " + area.getMapId() + " covers " + area.toHexString());
- if (area.getName() != null)
- System.out.print(' ' + area.getName());
- System.out.println();
- }
-
- if (kmlOutputFile != null) {
- File out = new File(kmlOutputFile);
- if (!out.isAbsolute())
- kmlOutputFile = new File(fileOutputDir, kmlOutputFile).getPath();
- System.out.println("Writing KML file to " + kmlOutputFile);
- areaList.writeKml(kmlOutputFile);
- }
-
- writeAreas(areas);
- writeArgsFile(areas);
- }
-
- private int getAreasPerPass(int areaCount) {
- return (int) Math.ceil((double) areaCount / (double) maxAreasPerPass);
- }
-
- /**
- * Deal with the command line arguments.
- */
- private void readArgs(String[] args) {
- ParamParser parser = new ParamParser();
- SplitterParams params = parser.parse(SplitterParams.class, args);
-
- if (!parser.getErrors().isEmpty()) {
- System.out.println();
- System.out.println("Invalid parameter(s):");
- for (String error : parser.getErrors()) {
- System.out.println(" " + error);
- }
- System.out.println();
- parser.displayUsage();
- System.exit(-1);
- }
-
- for (Map.Entry entry : parser.getConvertedParams().entrySet()) {
- String name = entry.getKey();
- Object value = entry.getValue();
- System.out.println(name + '=' + (value == null ? "" : value));
- }
-
- mapId = params.getMapid();
- overlapAmount = params.getOverlap();
- maxNodes = params.getMaxNodes();
- description = params.getDescription();
- geoNamesFile = params.getGeonamesFile();
- resolution = params.getResolution();
- trim = !params.isNoTrim();
- if (resolution < 1 || resolution > 24) {
- System.err.println("The --resolution parameter must be a value between 1 and 24. Resetting to 13.");
- resolution = 13;
- }
- mixed = params.isMixed();
- statusFreq = params.getStatusFreq();
-
- String outputDir = params.getOutputDir();
- fileOutputDir = new File(outputDir == null? DEFAULT_DIR: outputDir);
-
- maxAreasPerPass = params.getMaxAreas();
- if (maxAreasPerPass < 1 || maxAreasPerPass > 255) {
- System.err.println("The --max-areas parameter must be a value between 1 and 255. Resetting to 255.");
- maxAreasPerPass = 255;
- }
- kmlOutputFile = params.getWriteKml();
- densityMap = !params.isLegacyMode();
- if (!densityMap) {
- System.out.println("WARNING: Specifying --legacy-split will cause the first stage of the split to take much more memory! This option is considered deprecated and will be removed in a future build.");
- }
-
- maxThreads = params.getMaxThreads().getCount();
- filenames = parser.getAdditionalParams();
-
- String splitFile = params.getSplitFile();
- if (splitFile != null) {
- try {
- areaList = new AreaList();
- areaList.read(splitFile);
- areaList.dump();
- } catch (IOException e) {
- areaList = null;
- System.err.println("Could not read area list file");
- e.printStackTrace();
- }
- }
- }
-
- /**
- * Calculate the areas that we are going to split into by getting the total area and
- * then subdividing down until each area has at most max-nodes nodes in it.
- */
- private AreaList calculateAreas() throws IOException, XmlPullParserException {
-
- MapCollector nodes = densityMap ? new DensityMapCollector(trim, resolution) : new NodeCollector();
- MapProcessor processor = nodes;
-
- processMap(processor);
- //MapReader mapReader = processMap(processor);
-
- //System.out.print("A total of " + Utils.format(mapReader.getNodeCount()) + " nodes, " +
- // Utils.format(mapReader.getWayCount()) + " ways and " +
- // Utils.format(mapReader.getRelationCount()) + " relations were processed ");
-
- System.out.println("in " + filenames.size() + (filenames.size() == 1 ? " file" : " files"));
-
- //System.out.println("Min node ID = " + mapReader.getMinNodeId());
- //System.out.println("Max node ID = " + mapReader.getMaxNodeId());
-
- System.out.println("Time: " + new Date());
-
- Area exactArea = nodes.getExactArea();
- SplittableArea splittableArea = nodes.getRoundedArea(resolution);
- System.out.println("Exact map coverage is " + exactArea);
- System.out.println("Trimmed and rounded map coverage is " + splittableArea.getBounds());
- System.out.println("Splitting nodes into areas containing a maximum of " + Utils.format(maxNodes) + " nodes each...");
-
- List areas = splittableArea.split(maxNodes);
- return new AreaList(areas);
- }
-
- private void nameAreas() throws IOException {
- CityFinder cityFinder;
- if (geoNamesFile != null) {
- CityLoader cityLoader = new CityLoader(true);
- List cities = cityLoader.load(geoNamesFile);
- cityFinder = new DefaultCityFinder(cities);
- } else {
- cityFinder = new DummyCityFinder();
- }
-
- for (Area area : areaList.getAreas()) {
- // Decide what to call the area
- Set found = cityFinder.findCities(area);
- City bestMatch = null;
- for (City city : found) {
- if (bestMatch == null || city.getPopulation() > bestMatch.getPopulation()) {
- bestMatch = city;
- }
- }
- if (bestMatch != null)
- area.setName(bestMatch.getCountryCode() + '-' + bestMatch.getName());
- else
- area.setName(description);
- }
- }
-
- /**
- * Second pass, we have the areas so parse the file(s) again and write out each element
- * to the file(s) that should contain it.
- *
- * @param areaList Area list determined on the first pass.
- */
- private void writeAreas(List areas) throws IOException, XmlPullParserException {
- System.out.println("Writing out split osm files " + new Date());
-
- int numPasses = getAreasPerPass(areas.size());
- int areasPerPass = (int) Math.ceil((double) areas.size() / (double) numPasses);
-
- if (numPasses > 1) {
- System.out.println("Processing " + areas.size() + " areas in " + numPasses + " passes, " + areasPerPass + " areas at a time");
- } else {
- System.out.println("Processing " + areas.size() + " areas in a single pass");
- }
-
- for (int i = 0; i < numPasses; i++) {
- OSMWriter[] currentWriters = new OSMWriter[Math.min(areasPerPass, areas.size() - i * areasPerPass)];
- for (int j = 0; j < currentWriters.length; j++) {
- Area area = areas.get(i * areasPerPass + j);
- currentWriters[j] = new OSMWriter(area, fileOutputDir);
- currentWriters[j].initForWrite(area.getMapId(), overlapAmount);
- }
-
- System.out.println("Starting pass " + (i + 1) + " of " + numPasses + ", processing " + currentWriters.length +
- " areas (" + areas.get(i * areasPerPass).getMapId() + " to " +
- areas.get(i * areasPerPass + currentWriters.length - 1).getMapId() + ')');
-
- MapProcessor processor = new SplitProcessor(currentWriters, maxThreads);
- processMap(processor);
- //System.out.println("Wrote " + Utils.format(mapReader.getNodeCount()) + " nodes, " +
- // Utils.format(mapReader.getWayCount()) + " ways, " +
- // Utils.format(mapReader.getRelationCount()) + " relations");
- }
- }
-
- private void processMap(MapProcessor processor) throws XmlPullParserException {
- // Create both an XML reader and a binary reader, Dispatch each input to the
- // Appropriate parser.
- OSMParser parser = new OSMParser(processor, mixed);
- if (useStdIn) {
- System.out.println("Reading osm data from stdin...");
- Reader reader = new InputStreamReader(System.in, Charset.forName("UTF-8"));
- parser.setReader(reader);
- try {
- try {
- parser.parse();
- } finally {
- reader.close();
- }
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- for (String filename : filenames) {
- System.out.println("Processing " + filename);
- try {
- if (filename.endsWith(".osm.pbf")) {
- // Is it a binary file?
- File file = new File(filename);
- BlockInputStream blockinput = (new BlockInputStream(
- new FileInputStream(file), new BinaryMapParser(processor)));
- try {
- blockinput.process();
- } finally {
- blockinput.close();
- }
- } else {
- // No, try XML.
- Reader reader = Utils.openFile(filename, maxThreads > 1);
- parser.setReader(reader);
- try {
- parser.parse();
- } finally {
- reader.close();
- }
- }
- } catch (FileNotFoundException e) {
- e.printStackTrace();
- } catch (XmlPullParserException e) {
- e.printStackTrace();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- processor.endMap();
- }
-
- /**
- * Write a file that can be given to mkgmap that contains the correct arguments
- * for the split file pieces. You are encouraged to edit the file and so it
- * contains a template of all the arguments that you might want to use.
- */
- protected void writeArgsFile(List areas) {
- PrintWriter w;
- try {
- w = new PrintWriter(new FileWriter(new File(fileOutputDir, "template.args")));
- } catch (IOException e) {
- System.err.println("Could not write template.args file");
- return;
- }
-
- w.println("#");
- w.println("# This file can be given to mkgmap using the -c option");
- w.println("# Please edit it first to add a description of each map.");
- w.println("#");
- w.println();
-
- w.println("# You can set the family id for the map");
- w.println("# family-id: 980");
- w.println("# product-id: 1");
-
- w.println();
- w.println("# Following is a list of map tiles. Add a suitable description");
- w.println("# for each one.");
- for (Area a : areas) {
- w.println();
- w.format("mapname: %08d\n", a.getMapId());
- if (a.getName() == null)
- w.println("# description: OSM Map");
- else
- w.println("description: " + a.getName());
- w.format("input-file: %08d.osm.gz\n", a.getMapId());
- }
-
- w.println();
- w.close();
- }
-}
+/*
+ * Copyright (c) 2009.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 3 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ */
+
+package uk.me.parabola.splitter;
+
+import crosby.binary.file.BlockInputStream;
+import org.xmlpull.v1.XmlPullParserException;
+import uk.me.parabola.splitter.args.ParamParser;
+import uk.me.parabola.splitter.args.SplitterParams;
+import uk.me.parabola.splitter.geo.City;
+import uk.me.parabola.splitter.geo.CityFinder;
+import uk.me.parabola.splitter.geo.CityLoader;
+import uk.me.parabola.splitter.geo.DefaultCityFinder;
+import uk.me.parabola.splitter.geo.DummyCityFinder;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.PrintWriter;
+import java.io.Reader;
+import java.nio.charset.Charset;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Splitter for OSM files with the purpose of providing input files for mkgmap.
+ *
+ * The input file is split so that no piece has more than a given number of nodes in it.
+ *
+ * @author Steve Ratcliffe
+ */
+public class Main {
+ private static final String DEFAULT_DIR = ".";
+
+ // We can only process a maximum of 255 areas at a time because we
+ // compress an area ID into 8 bits to save memory (and 0 is reserved)
+ private int maxAreasPerPass;
+
+ // A list of the OSM files to parse.
+ private List filenames;
+
+ // The description to write into the template.args file.
+ private String description;
+
+ // The starting map ID.
+ private int mapId;
+
+ // The amount in map units that tiles overlap (note that the final img's will not overlap
+ // but the input files do).
+ private int overlapAmount;
+
+ // The max number of nodes that will appear in a single file.
+ private int maxNodes;
+
+ // The maximum resolution of the map to be produced by mkgmap. This is a value in the range
+ // 0-24. Higher numbers mean higher detail. The resolution determines how the tiles must
+ // be aligned. Eg a resolution of 13 means the tiles need to have their edges aligned to
+ // multiples of 2 ^ (24 - 13) = 2048 map units, and their widths and heights must be a multiple
+ // of 2 * 2 ^ (24 - 13) = 4096 units. The tile widths and height multiples are double the tile
+ // alignment because the center point of the tile is stored, and that must be aligned the
+ // same as the tile edges are.
+ private int resolution;
+
+ // Whether or not to trim tiles of any empty space around their edges.
+ private boolean trim;
+ // This gets set if no osm file is supplied as a parameter and the cache is empty.
+ private boolean useStdIn;
+ // Set if there is a previous area file given on the command line.
+ private AreaList areaList;
+ // Whether or not the source OSM file(s) contain strictly nodes first, then ways, then rels,
+ // or they're all mixed up. Running with mixed enabled takes longer.
+ private boolean mixed;
+ // The path where the results are written out to.
+ private File fileOutputDir;
+ // A GeoNames file to use for naming the tiles.
+ private String geoNamesFile;
+ // How often (in seconds) to provide JVM status information. Zero = no information.
+ private int statusFreq;
+ // Whether to use the density map. Disabling this (not recommended) causes the splitter to
+ // revert to using legacy mode which takes MUCH more memory during phase one.
+ private boolean densityMap;
+
+ private String kmlOutputFile;
+ // The maximum number of threads the splitter should use.
+ private int maxThreads;
+ // The output type
+ private boolean pbfOutput;
+
+ public static void main(String[] args) {
+
+ Main m = new Main();
+ m.start(args);
+ }
+
+ private void start(String[] args) {
+ readArgs(args);
+ if (statusFreq > 0) {
+ JVMHealthMonitor.start(statusFreq);
+ }
+ long start = System.currentTimeMillis();
+ System.out.println("Time started: " + new Date());
+ try {
+ split();
+ } catch (IOException e) {
+ System.err.println("Error opening or reading file " + e);
+ e.printStackTrace();
+ } catch (XmlPullParserException e) {
+ System.err.println("Error parsing xml from file " + e);
+ e.printStackTrace();
+ }
+ System.out.println("Time finished: " + new Date());
+ System.out.println("Total time taken: " + (System.currentTimeMillis() - start) / 1000 + 's');
+ }
+
+ private void split() throws IOException, XmlPullParserException {
+
+ File outputDir = fileOutputDir;
+ if (!outputDir.exists()) {
+ System.out.println("Output directory not found. Creating directory '" + fileOutputDir + "'");
+ if (!outputDir.mkdirs()) {
+ System.err.println("Unable to create output directory! Using default directory instead");
+ fileOutputDir = new File(DEFAULT_DIR);
+ }
+ } else if (!outputDir.isDirectory()) {
+ System.err.println("The --output-dir parameter must specify a directory. The --output-dir parameter is being ignored, writing to default directory instead.");
+ fileOutputDir = new File(DEFAULT_DIR);
+ }
+
+ if (filenames.isEmpty()) {
+ if (areaList == null) {
+ throw new IllegalArgumentException("No .osm files were supplied so at least one of --cache or --split-file must be specified");
+ } else {
+ int areaCount = areaList.getAreas().size();
+ int passes = getAreasPerPass(areaCount);
+ if (passes > 1) {
+ throw new IllegalArgumentException("No .osm files or --cache parameter were supplied, but stdin cannot be used because " + passes
+ + " passes are required to write out the areas. Either provide --cache or increase --max-areas to match the number of areas (" + areaCount + ')');
+ }
+ useStdIn = true;
+ }
+ }
+
+ if (areaList == null) {
+ int alignment = 1 << (24 - resolution);
+ System.out.println("Map is being split for resolution " + resolution + ':');
+ System.out.println(" - area boundaries are aligned to 0x" + Integer.toHexString(alignment) + " map units");
+ System.out.println(" - areas are multiples of 0x" + Integer.toHexString(alignment * 2) + " map units wide and high");
+ areaList = calculateAreas();
+ for (Area area : areaList.getAreas()) {
+ area.setMapId(mapId++);
+ }
+ nameAreas();
+ areaList.write(new File(fileOutputDir, "areas.list").getPath());
+ } else {
+ nameAreas();
+ }
+
+ List areas = areaList.getAreas();
+ System.out.println(areas.size() + " areas:");
+ for (Area area : areas) {
+ System.out.print("Area " + area.getMapId() + " covers " + area.toHexString());
+ if (area.getName() != null)
+ System.out.print(' ' + area.getName());
+ System.out.println();
+ }
+
+ if (kmlOutputFile != null) {
+ File out = new File(kmlOutputFile);
+ if (!out.isAbsolute())
+ kmlOutputFile = new File(fileOutputDir, kmlOutputFile).getPath();
+ System.out.println("Writing KML file to " + kmlOutputFile);
+ areaList.writeKml(kmlOutputFile);
+ }
+
+ writeAreas(areas);
+ writeArgsFile(areas);
+ }
+
+ private int getAreasPerPass(int areaCount) {
+ return (int) Math.ceil((double) areaCount / (double) maxAreasPerPass);
+ }
+
+ /**
+ * Deal with the command line arguments.
+ */
+ private void readArgs(String[] args) {
+ ParamParser parser = new ParamParser();
+ SplitterParams params = parser.parse(SplitterParams.class, args);
+
+ if (!parser.getErrors().isEmpty()) {
+ System.out.println();
+ System.out.println("Invalid parameter(s):");
+ for (String error : parser.getErrors()) {
+ System.out.println(" " + error);
+ }
+ System.out.println();
+ parser.displayUsage();
+ System.exit(-1);
+ }
+
+ for (Map.Entry entry : parser.getConvertedParams().entrySet()) {
+ String name = entry.getKey();
+ Object value = entry.getValue();
+ System.out.println(name + '=' + (value == null ? "" : value));
+ }
+
+ mapId = params.getMapid();
+ overlapAmount = params.getOverlap();
+ maxNodes = params.getMaxNodes();
+ description = params.getDescription();
+ geoNamesFile = params.getGeonamesFile();
+ resolution = params.getResolution();
+ trim = !params.isNoTrim();
+ pbfOutput = params.isPbf();
+
+ if (resolution < 1 || resolution > 24) {
+ System.err.println("The --resolution parameter must be a value between 1 and 24. Resetting to 13.");
+ resolution = 13;
+ }
+ mixed = params.isMixed();
+ statusFreq = params.getStatusFreq();
+
+ String outputDir = params.getOutputDir();
+ fileOutputDir = new File(outputDir == null? DEFAULT_DIR: outputDir);
+
+ maxAreasPerPass = params.getMaxAreas();
+ if (maxAreasPerPass < 1 || maxAreasPerPass > 255) {
+ System.err.println("The --max-areas parameter must be a value between 1 and 255. Resetting to 255.");
+ maxAreasPerPass = 255;
+ }
+ kmlOutputFile = params.getWriteKml();
+ densityMap = !params.isLegacyMode();
+ if (!densityMap) {
+ System.out.println("WARNING: Specifying --legacy-split will cause the first stage of the split to take much more memory! This option is considered deprecated and will be removed in a future build.");
+ }
+
+ maxThreads = params.getMaxThreads().getCount();
+ filenames = parser.getAdditionalParams();
+
+ String splitFile = params.getSplitFile();
+ if (splitFile != null) {
+ try {
+ areaList = new AreaList();
+ areaList.read(splitFile);
+ areaList.dump();
+ } catch (IOException e) {
+ areaList = null;
+ System.err.println("Could not read area list file");
+ e.printStackTrace();
+ }
+ }
+ }
+
+ /**
+ * Calculate the areas that we are going to split into by getting the total area and
+ * then subdividing down until each area has at most max-nodes nodes in it.
+ */
+ private AreaList calculateAreas() throws IOException, XmlPullParserException {
+
+ MapCollector nodes = densityMap ? new DensityMapCollector(trim, resolution) : new NodeCollector();
+ MapProcessor processor = nodes;
+
+ processMap(processor);
+ //MapReader mapReader = processMap(processor);
+
+ //System.out.print("A total of " + Utils.format(mapReader.getNodeCount()) + " nodes, " +
+ // Utils.format(mapReader.getWayCount()) + " ways and " +
+ // Utils.format(mapReader.getRelationCount()) + " relations were processed ");
+
+ System.out.println("in " + filenames.size() + (filenames.size() == 1 ? " file" : " files"));
+
+ //System.out.println("Min node ID = " + mapReader.getMinNodeId());
+ //System.out.println("Max node ID = " + mapReader.getMaxNodeId());
+
+ System.out.println("Time: " + new Date());
+
+ Area exactArea = nodes.getExactArea();
+ SplittableArea splittableArea = nodes.getRoundedArea(resolution);
+ System.out.println("Exact map coverage is " + exactArea);
+ System.out.println("Trimmed and rounded map coverage is " + splittableArea.getBounds());
+ System.out.println("Splitting nodes into areas containing a maximum of " + Utils.format(maxNodes) + " nodes each...");
+
+ List areas = splittableArea.split(maxNodes);
+ return new AreaList(areas);
+ }
+
+ private void nameAreas() throws IOException {
+ CityFinder cityFinder;
+ if (geoNamesFile != null) {
+ CityLoader cityLoader = new CityLoader(true);
+ List cities = cityLoader.load(geoNamesFile);
+ cityFinder = new DefaultCityFinder(cities);
+ } else {
+ cityFinder = new DummyCityFinder();
+ }
+
+ for (Area area : areaList.getAreas()) {
+ // Decide what to call the area
+ Set found = cityFinder.findCities(area);
+ City bestMatch = null;
+ for (City city : found) {
+ if (bestMatch == null || city.getPopulation() > bestMatch.getPopulation()) {
+ bestMatch = city;
+ }
+ }
+ if (bestMatch != null)
+ area.setName(bestMatch.getCountryCode() + '-' + bestMatch.getName());
+ else
+ area.setName(description);
+ }
+ }
+
+ /**
+ * Second pass, we have the areas so parse the file(s) again and write out each element
+ * to the file(s) that should contain it.
+ *
+ * @param areaList Area list determined on the first pass.
+ */
+ private void writeAreas(List areas) throws IOException, XmlPullParserException {
+ System.out.println("Writing out split osm files " + new Date());
+
+ int numPasses = getAreasPerPass(areas.size());
+ int areasPerPass = (int) Math.ceil((double) areas.size() / (double) numPasses);
+
+ if (numPasses > 1) {
+ System.out.println("Processing " + areas.size() + " areas in " + numPasses + " passes, " + areasPerPass + " areas at a time");
+ } else {
+ System.out.println("Processing " + areas.size() + " areas in a single pass");
+ }
+
+ for (int i = 0; i < numPasses; i++) {
+ OSMWriter[] currentWriters = new OSMWriter[Math.min(areasPerPass, areas.size() - i * areasPerPass)];
+ for (int j = 0; j < currentWriters.length; j++) {
+ Area area = areas.get(i * areasPerPass + j);
+ currentWriters[j] = pbfOutput ? new BinaryMapWriter(area, fileOutputDir) : new OSMXMLWriter(area, fileOutputDir);
+ currentWriters[j].initForWrite(area.getMapId(), overlapAmount);
+ }
+
+ System.out.println("Starting pass " + (i + 1) + " of " + numPasses + ", processing " + currentWriters.length +
+ " areas (" + areas.get(i * areasPerPass).getMapId() + " to " +
+ areas.get(i * areasPerPass + currentWriters.length - 1).getMapId() + ')');
+
+ MapProcessor processor = new SplitProcessor(currentWriters, maxThreads);
+ processMap(processor);
+ //System.out.println("Wrote " + Utils.format(mapReader.getNodeCount()) + " nodes, " +
+ // Utils.format(mapReader.getWayCount()) + " ways, " +
+ // Utils.format(mapReader.getRelationCount()) + " relations");
+ }
+ }
+
+ private void processMap(MapProcessor processor) throws XmlPullParserException {
+ // Create both an XML reader and a binary reader, Dispatch each input to the
+ // Appropriate parser.
+ OSMParser parser = new OSMParser(processor, mixed);
+ if (useStdIn) {
+ System.out.println("Reading osm data from stdin...");
+ Reader reader = new InputStreamReader(System.in, Charset.forName("UTF-8"));
+ parser.setReader(reader);
+ try {
+ try {
+ parser.parse();
+ } finally {
+ reader.close();
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ for (String filename : filenames) {
+ System.out.println("Processing " + filename);
+ try {
+ if (filename.endsWith(".osm.pbf")) {
+ // Is it a binary file?
+ File file = new File(filename);
+ BlockInputStream blockinput = (new BlockInputStream(
+ new FileInputStream(file), new BinaryMapParser(processor)));
+ try {
+ blockinput.process();
+ } finally {
+ blockinput.close();
+ }
+ } else {
+ // No, try XML.
+ Reader reader = Utils.openFile(filename, maxThreads > 1);
+ parser.setReader(reader);
+ try {
+ parser.parse();
+ } finally {
+ reader.close();
+ }
+ }
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
+ } catch (XmlPullParserException e) {
+ e.printStackTrace();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ processor.endMap();
+ }
+
+ /**
+ * Write a file that can be given to mkgmap that contains the correct arguments
+ * for the split file pieces. You are encouraged to edit the file and so it
+ * contains a template of all the arguments that you might want to use.
+ */
+ protected void writeArgsFile(List areas) {
+ PrintWriter w;
+ try {
+ w = new PrintWriter(new FileWriter(new File(fileOutputDir, "template.args")));
+ } catch (IOException e) {
+ System.err.println("Could not write template.args file");
+ return;
+ }
+
+ w.println("#");
+ w.println("# This file can be given to mkgmap using the -c option");
+ w.println("# Please edit it first to add a description of each map.");
+ w.println("#");
+ w.println();
+
+ w.println("# You can set the family id for the map");
+ w.println("# family-id: 980");
+ w.println("# product-id: 1");
+
+ w.println();
+ w.println("# Following is a list of map tiles. Add a suitable description");
+ w.println("# for each one.");
+ for (Area a : areas) {
+ w.println();
+ w.format("mapname: %08d\n", a.getMapId());
+ if (a.getName() == null)
+ w.println("# description: OSM Map");
+ else
+ w.println("description: " + a.getName());
+ if(pbfOutput)
+ w.format("input-file: %08d.osm.pbf\n", a.getMapId());
+ else
+ w.format("input-file: %08d.osm.gz\n", a.getMapId());
+ }
+
+ w.println();
+ w.close();
+ }
+}
Index: src/uk/me/parabola/splitter/JVMHealthMonitor.java
===================================================================
--- src/uk/me/parabola/splitter/JVMHealthMonitor.java (revision 171)
+++ src/uk/me/parabola/splitter/JVMHealthMonitor.java (working copy)
@@ -38,7 +38,6 @@
}
startTime = System.currentTimeMillis();
statusThread = new Thread(new Runnable() {
- @Override
public void run() {
int iter=0;
while (true) {
Index: src/uk/me/parabola/splitter/OSMWriter.java
===================================================================
--- src/uk/me/parabola/splitter/OSMWriter.java (revision 171)
+++ src/uk/me/parabola/splitter/OSMWriter.java (working copy)
@@ -1,252 +1,48 @@
-/*
- * Copyright (c) 2009.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 3 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * General Public License for more details.
- */
-
-package uk.me.parabola.splitter;
-
-import it.unimi.dsi.fastutil.ints.IntArrayList;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.OutputStreamWriter;
-import java.io.Writer;
-import java.text.DecimalFormat;
-import java.text.DecimalFormatSymbols;
-import java.util.Formatter;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Locale;
-import java.util.zip.GZIPOutputStream;
-
-public class OSMWriter {
- private final DecimalFormat numberFormat = new DecimalFormat(
- "0.#######;-0.#######",
- new DecimalFormatSymbols(Locale.US)
- );
-
- private final Area bounds;
- private Writer writer;
- private Area extendedBounds;
- private File outputDir;
-
-
- public OSMWriter(Area bounds, File outputDir) {
- this.bounds = bounds;
- this.outputDir = outputDir;
- }
-
- public Area getExtendedBounds() {
- return extendedBounds;
- }
-
- public void initForWrite(int mapId, int extra) {
- extendedBounds = new Area(bounds.getMinLat() - extra,
- bounds.getMinLong() - extra,
- bounds.getMaxLat() + extra,
- bounds.getMaxLong() + extra);
-
- String filename = new Formatter().format(Locale.ROOT, "%08d.osm.gz", mapId).toString();
- try {
- FileOutputStream fos = new FileOutputStream(new File(outputDir, filename));
- OutputStream zos = new GZIPOutputStream(fos);
- writer = new OutputStreamWriter(zos, "utf-8");
- writeHeader();
- } catch (IOException e) {
- System.out.println("Could not open or write file header. Reason: " + e.getMessage());
- throw new RuntimeException(e);
- }
- }
-
- private void writeHeader() throws IOException {
- writeString("\n");
- writeString("\n");
-
- writeString("\n");
- }
-
- public void finishWrite() {
- try {
- writeString("\n");
- flush();
- writer.close();
- } catch (IOException e) {
- System.out.println("Could not write end of file: " + e);
- }
- }
-
- public boolean nodeBelongsToThisArea(Node node) {
- return (extendedBounds.contains(node.getMapLat(), node.getMapLon()));
- }
-
- public void write(Node node) throws IOException {
- writeString("\n");
- writeTags(node);
- writeString("\n");
- } else {
- writeString("'/>\n");
- }
- }
-
- public void write(Way way) throws IOException {
- writeString("\n");
- IntArrayList refs = way.getRefs();
- for (int i = 0; i < refs.size(); i++) {
- writeString("\n");
- }
- if (way.hasTags())
- writeTags(way);
- writeString("\n");
- }
-
- public void write(Relation rel) throws IOException {
- writeString("\n");
- List memlist = rel.getMembers();
- for (Relation.Member m : memlist) {
- if (m.getType() == null || m.getRef() == 0) {
- System.err.println("Invalid relation member found in relation " + rel.getId() + ": member type=" + m.getType() + ", ref=" + m.getRef() + ", role=" + m.getRole() + ". Ignoring this member");
- continue;
- }
- writeString("\n");
- }
- if (rel.hasTags())
- writeTags(rel);
- writeString("\n");
- }
-
- private void writeTags(Element element) throws IOException {
- Iterator it = element.tagsIterator();
- while (it.hasNext()) {
- Element.Tag entry = it.next();
- writeString("\n");
- }
- }
-
- private void writeAttribute(String value) throws IOException {
- for (int i = 0; i < value.length(); i++) {
- char c = value.charAt(i);
- switch (c) {
- case '\'':
- writeString("'");
- break;
- case '&':
- writeString("&");
- break;
- case '<':
- writeString("<");
- break;
- default:
- writeChar(c);
- }
- }
- }
-
- private int index;
- private final char[] charBuf = new char[4096];
-
- private void checkFlush(int i) throws IOException {
- if (charBuf.length - index < i) {
- flush();
- }
- }
-
- private void flush() throws IOException {
- writer.write(charBuf, 0, index);
- index = 0;
- }
-
- private void writeString(String value) throws IOException {
- int start = 0;
- int end = value.length();
- int len;
- while ((len = charBuf.length - index) < end - start) {
- value.getChars(start, start + len, charBuf, index);
- start += len;
- index = charBuf.length;
- flush();
- }
- value.getChars(start, end, charBuf, index);
- index += end - start;
- }
-
- /** Write a double to full precision */
- private void writeLongDouble(double value) throws IOException {
- checkFlush(22);
- writeString(Double.toString(value));
- }
- /** Write a double truncated to OSM's 7 digits of precision
- *
- * TODO: Optimize. Responsible for >30% of the runtime after other using binary
- * format and improved hash table.
- */
- private void writeDouble(double value) throws IOException {
- checkFlush(22);
- // Punt on some annoying specialcases
- if (value < -200 || value > 200 || (value > -1 && value < 1))
- writeString(numberFormat.format(value));
- else {
- if (value < 0) {
- charBuf[index++] = '-'; // Write directly.
- value = -value;
- }
-
- int val = (int)Math.round(value*10000000);
- StringBuilder s = new StringBuilder(Integer.toString(val));
- s.insert(s.length()-7, '.');
- writeString(s.toString());
- }
- }
-
- private void writeInt(int value) throws IOException {
- checkFlush(11);
- index += Convert.intToString(value, charBuf, index);
- }
-
- private void writeChar(char value) throws IOException {
- checkFlush(1);
- charBuf[index++] = value;
- }
-}
+/*
+ * Copyright (c) 2009.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 3 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ */
+
+package uk.me.parabola.splitter;
+
+import java.io.File;
+import java.io.IOException;
+
+public abstract class OSMWriter {
+
+ protected final Area bounds;
+ protected Area extendedBounds;
+ protected File outputDir;
+
+
+ public OSMWriter(Area bounds, File outputDir) {
+ this.bounds = bounds;
+ this.outputDir = outputDir;
+ }
+
+ public Area getExtendedBounds() {
+ return extendedBounds;
+ }
+
+ public abstract void initForWrite(int mapId, int extra);
+
+ public abstract void finishWrite();
+
+ public boolean nodeBelongsToThisArea(Node node) {
+ return (extendedBounds.contains(node.getMapLat(), node.getMapLon()));
+ }
+
+ public abstract void write(Node node) throws IOException;
+
+ public abstract void write(Way way) throws IOException;
+
+ public abstract void write(Relation rel) throws IOException;
+}
Index: src/uk/me/parabola/splitter/NodeCollector.java
===================================================================
--- src/uk/me/parabola/splitter/NodeCollector.java (revision 171)
+++ src/uk/me/parabola/splitter/NodeCollector.java (working copy)
@@ -22,12 +22,10 @@
private final MapDetails details = new MapDetails();
private Area bounds;
- @Override
public boolean isStartNodeOnly() {
return true;
}
- @Override
public void boundTag(Area bounds) {
if (this.bounds == null)
this.bounds = bounds;
@@ -35,7 +33,6 @@
this.bounds = this.bounds.add(bounds);
}
- @Override
public void processNode(Node n) {
// Since we are rounding areas to fit on a low zoom boundary we
// can drop the bottom 8 bits of the lat and lon and then fit
@@ -48,16 +45,12 @@
details.addToBounds(glat, glon);
}
- @Override
public void processWay(Way w) {}
- @Override
public void processRelation(Relation r) {}
- @Override
public void endMap() {}
- @Override
public Area getExactArea() {
if (bounds != null) {
return bounds;
@@ -66,7 +59,6 @@
}
}
- @Override
public SplittableArea getRoundedArea(int resolution) {
Area bounds = RoundingUtils.round(getExactArea(), resolution);
SplittableArea result = new SplittableNodeArea(bounds, coords, resolution);