August 2009 - mkgmap-dev - The mkgmap lists

[PATCH v3] sea polygons
by Christian Gawron 10 Aug '09

10 Aug '09

Here is an improved version of the sea polygon patch which also handles shorelines intersecting the boundary of the map (set by the <bounds> element). I have tested it with Ireland - there are still some islands which are "flooded", so the patch should not be considered "final". Best wishes Christian Index: src/uk/me/parabola/mkgmap/reader/osm/Way.java =================================================================== --- src/uk/me/parabola/mkgmap/reader/osm/Way.java (Revision 1124) +++ src/uk/me/parabola/mkgmap/reader/osm/Way.java (Arbeitskopie) @@ -76,6 +76,10 @@ } } + public boolean isClosed() { + return points.get(0).equals(points.get(points.size()-1)); + } + /** * A simple representation of this way. * @return A string with the name and start point Index: src/uk/me/parabola/mkgmap/reader/osm/MultiPolygonRelation.java =================================================================== --- src/uk/me/parabola/mkgmap/reader/osm/MultiPolygonRelation.java (Revision 1124) +++ src/uk/me/parabola/mkgmap/reader/osm/MultiPolygonRelation.java (Arbeitskopie) @@ -1,11 +1,10 @@ package uk.me.parabola.mkgmap.reader.osm; import java.util.ArrayList; -import java.util.Collection; +import java.util.Iterator; import java.util.List; import java.util.Map; -import uk.me.parabola.imgfmt.Utils; import uk.me.parabola.imgfmt.app.Coord; /** @@ -16,15 +15,19 @@ */ public class MultiPolygonRelation extends Relation { private Way outer; - private final Collection<Way> inners = new ArrayList<Way>(); + private List<Way> outers = new ArrayList<Way>(); + private List<Way> inners = new ArrayList<Way>(); + private Map<Long, Way> myWayMap; /** * Create an instance based on an exsiting relation. We need to do * this because the type of the relation is not known until after all * its tags are read in. * @param other The relation to base this one on. + * @param wayMap Map of all ways. */ - public MultiPolygonRelation(Relation other) { + public MultiPolygonRelation(Relation other, Map<Long, Way> wayMap) { + myWayMap = wayMap; setId(other.getId()); for (Map.Entry<Element, String> pairs: other.getRoles().entrySet()){ addElement(pairs.getValue(), pairs.getKey()); @@ -33,10 +36,12 @@ if (value != null && pairs.getKey() instanceof Way) { Way way = (Way) pairs.getKey(); - if (value.equals("outer")) - outer = way; - else if (value.equals("inner")) + if (value.equals("outer")){ + outers.add(way); + } + else if (value.equals("inner")){ inners.add(way); + } } } @@ -45,18 +50,63 @@ } /** Process the ways in this relation. + * Joins way with the role "outer" * Adds ways with the role "inner" to the way with the role "outer" */ public void processElements() { - if (outer != null) - { + + if (outers != null) + { + // copy first outer way + Iterator<Way> it = outers.iterator(); + if (it.hasNext()){ + // duplicate outer way and remove tags for cascaded multipolygons + Way tempWay = it.next(); + outer = new Way(-tempWay.getId()); + outer.copyTags(tempWay); + for(Coord point: tempWay.getPoints()){ + outer.addPoint(point); + } + myWayMap.put(outer.getId(), outer); + if (tempWay.getTags() != null){ + tempWay.getTags().removeAll(); + } + it.remove(); + } + + // if we have more than one outer way, we join them if they are parts of a long way + it = outers.iterator(); + while (it.hasNext()){ + Way tempWay = it.next(); + if (tempWay.getPoints().get(0) == outer.getPoints().get(outer.getPoints().size()-1)){ + for(Coord point: tempWay.getPoints()){ + outer.addPoint(point); + } + if (tempWay.getTags() != null){ + tempWay.getTags().removeAll(); + } + it.remove(); + it = outers.iterator(); + } + } + for (Way w: inners) { - if (w != null) { - List<Coord> pts = w.getPoints(); - int[] insert = findCpa(outer.getPoints(), pts); - if (insert[0] > 0) - insertPoints(pts, insert[0], insert[1]); - pts.clear(); + if (w != null && outer!= null) { + int[] insert = findCpa(outer.getPoints(), w.getPoints()); + insertPoints(w, insert[0], insert[1]); + + // remove tags from inner way that are available in the outer way + if (outer.getTags() != null){ + for (Map.Entry<String, String> mapTags: outer.getTags().getKeyValues().entrySet()){ + String key = mapTags.getKey(); + String value = mapTags.getValue(); + if (w.getTag(key) != null){ + if (w.getTag(key).equals(value)){ + w.deleteTag(key); + } + } + } + } } } } @@ -64,22 +114,43 @@ /** * Insert Coordinates into the outer way. - * @param inList List of Coordinates to be inserted + * @param way Way to be inserted * @param out Coordinates will be inserted after this point in the outer way. * @param in Points will be inserted starting at this index, * then from element 0 to (including) this element; */ - private void insertPoints(List<Coord> inList, int out, int in){ + private void insertPoints(Way way, int out, int in){ List<Coord> outList = outer.getPoints(); + List<Coord> inList = way.getPoints(); int index = out+1; - for (int i = in; i < inList.size(); i++) + for (int i = in; i < inList.size(); i++){ outList.add(index++, inList.get(i)); - for (int i = 0; i <= in; i++) + } + for (int i = 0; i < in; i++){ outList.add(index++, inList.get(i)); - - //with this line commented we get triangles, when uncommented some areas disappear - // at least in mapsource, on device itself looks OK. - outList.add(index,outList.get(out)); + } + + if (outer.getPoints().size() < 32){ + outList.add(index++, inList.get(in)); + outList.add(index, outList.get(out)); + } + else{ + // we shift the nodes to avoid duplicate nodes (large areas only) + int oLat = outList.get(out).getLatitude(); + int oLon = outList.get(out).getLongitude(); + int iLat = inList.get(in).getLatitude(); + int iLon = inList.get(in).getLongitude(); + if (Math.abs(oLat - iLat) > Math.abs(oLon - iLon)){ + int delta = (oLon > iLon)? -1 : 1; + outList.add(index++, new Coord(iLat + delta, iLon)); + outList.add(index, new Coord(oLat + delta, oLon)); + } + else{ + int delta = (oLat > iLat)? 1 : -1; + outList.add(index++, new Coord(iLat, iLon + delta)); + outList.add(index, new Coord(oLat, oLon + delta)); + } + } } /** Index: src/uk/me/parabola/mkgmap/reader/osm/xml/Osm5XmlHandler.java =================================================================== --- src/uk/me/parabola/mkgmap/reader/osm/xml/Osm5XmlHandler.java (Revision 1124) +++ src/uk/me/parabola/mkgmap/reader/osm/xml/Osm5XmlHandler.java (Arbeitskopie) @@ -19,9 +19,12 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.IdentityHashMap; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.SortedMap; +import java.util.TreeMap; import uk.me.parabola.imgfmt.app.Area; import uk.me.parabola.imgfmt.app.Coord; @@ -65,6 +68,7 @@ private final Map<String, Long> fakeIdMap = new HashMap<String, Long>(); private final List<Node> exits = new ArrayList<Node>(); private final List<Way> motorways = new ArrayList<Way>(); + private final List<Way> shoreline = new ArrayList<Way>(); private static final int MODE_NODE = 1; private static final int MODE_WAY = 2; @@ -92,6 +96,7 @@ private final boolean ignoreTurnRestrictions; private final boolean linkPOIsToWays; private final boolean routing; + private final boolean generateSea; private final Double minimumArcLength; private final String frigRoundabouts; @@ -105,6 +110,7 @@ } linkPOIsToWays = props.getProperty("link-pois-to-ways", false); ignoreBounds = props.getProperty("ignore-osm-bounds", false); + generateSea = props.getProperty("generate-sea", false); routing = props.containsKey("route"); String rsa = props.getProperty("remove-short-arcs", null); if(rsa != null) @@ -370,6 +376,8 @@ if("motorway".equals(highway) || "trunk".equals(highway)) motorways.add(currentWay); + if(generateSea && "coastline".equals(currentWay.getTag("natural"))) + shoreline.add(currentWay); currentWay = null; // ways are processed at the end of the document, // may be changed by a Relation class @@ -399,7 +407,7 @@ String type = currentRelation.getTag("type"); if (type != null) { if ("multipolygon".equals(type)) - currentRelation = new MultiPolygonRelation(currentRelation); + currentRelation = new MultiPolygonRelation(currentRelation, wayMap); else if("restriction".equals(type)) { if(ignoreTurnRestrictions) @@ -446,6 +454,10 @@ } coordMap = null; + + if (generateSea) + generateSeaPolygon(shoreline); + for (Relation r : relationMap.values()) converter.convertRelation(r); @@ -707,7 +719,7 @@ } currentWay.addPoint(co); co.incHighwayCount(); // nodes (way joins) will have highwayCount > 1 - if(minimumArcLength != null) + if (minimumArcLength != null || generateSea) nodeIdMap.put(co, id); } } @@ -746,4 +758,267 @@ return fakeIdVal; } } + + private void generateSeaPolygon(List<Way> shoreline) { + + Area seaBounds; + if (bbox != null) + seaBounds = bbox; + else + seaBounds = mapper.getBounds(); + + log.info("generating sea, seaBounds=", seaBounds); + int minLat = seaBounds.getMinLat(); + int maxLat = seaBounds.getMaxLat(); + int minLong = seaBounds.getMinLong(); + int maxLong = seaBounds.getMaxLong(); + Coord nw = new Coord(minLat, minLong); + Coord ne = new Coord(minLat, maxLong); + Coord sw = new Coord(maxLat, minLong); + Coord se = new Coord(maxLat, maxLong); + + long seaId = (1L << 62) + nextFakeId++; + Way sea = new Way(seaId); + sea.addPoint(nw); + sea.addPoint(sw); + sea.addPoint(se); + sea.addPoint(ne); + sea.addPoint(nw); + sea.addTag("natural", "sea"); + log.info("sea: ", sea); + wayMap.put(seaId, sea); + + long multiId = (1L << 62) + nextFakeId++; + Relation seaRelation = new GeneralRelation(multiId); + seaRelation.addTag("type", "multipolygon"); + seaRelation.addElement("outer", sea); + + List<Way> islands = new ArrayList(); + + // handle islands (closes shoreline components) first (they're easy) + Iterator<Way> it = shoreline.iterator(); + while (it.hasNext()) { + Way w = it.next(); + if (w.isClosed()) { + islands.add(w); + it.remove(); + } + } + concatenateWays(shoreline); + // there may be more islands now + it = shoreline.iterator(); + while (it.hasNext()) { + Way w = it.next(); + if (w.isClosed()) { + log.debug("island after concatenating\n"); + islands.add(w); + it.remove(); + } + } + + for (Way w : islands) { + seaRelation.addElement("inner", w); + } + + SortedMap<EdgeHit, Way> hitMap = new TreeMap<EdgeHit, Way>(); + for (Way w : shoreline) { + List<Coord> points = w.getPoints(); + Coord pStart = points.get(0); + Coord pEnd = points.get(points.size()-1); + + EdgeHit hStart = getEdgeHit(seaBounds, pStart); + EdgeHit hEnd = getEdgeHit(seaBounds, pEnd); + if (hStart == null || hEnd == null) { + log.error(String.format("Non-closed coastline segment does not hit bounding box - expect strange results: %d (%s) %d (%s) %s\n", + nodeIdMap.get(pStart), pStart.toDegreeString(), + nodeIdMap.get(pEnd), pEnd.toDegreeString(), + pStart.toOSMURL())); + } + else { + log.debug("hits: ", hStart, hEnd); + hitMap.put(hStart, w); + hitMap.put(hEnd, null); + } + } + + if (hitMap.size() > 0) { + Way w = new Way((1L << 62) + nextFakeId++); + Iterator<EdgeHit> it1; + it1=hitMap.keySet().iterator(); + EdgeHit hit = it1.next(); + while (hit != null) { + Way seg = hitMap.get(hit); + if (seg != null) { + log.info("adding shoreline ", w); + w.getPoints().addAll(seg.getPoints()); + if (it1.hasNext()) + hit = it1.next(); + else + hit = null; + } + else { + EdgeHit h1 = it1.next(); + Coord p; + p = hit.getPoint(seaBounds); + log.debug("way: ", hit, p); + w.addPoint(p); + if (hit.edge < h1.edge) { + log.info("joining: ", hit, h1); + for (int i=hit.edge; i<h1.edge; i++) { + hit = new EdgeHit(i, 1.0); + p = hit.getPoint(seaBounds); + log.debug("way: ", hit, p); + w.addPoint(p); + } + } + else if (hit.edge > h1.edge) { + log.info("joining: ", hit, h1); + for (int i=hit.edge; i<4; i++) { + hit = new EdgeHit(i, 1.0); + p = hit.getPoint(seaBounds); + log.debug("way: ", hit, p); + w.addPoint(p); + } + for (int i=0; i<h1.edge; i++) { + hit = new EdgeHit(i, 1.0); + p = hit.getPoint(seaBounds); + log.debug("way: ", hit, p); + w.addPoint(p); + } + } + w.addPoint(h1.getPoint(seaBounds)); + hit = h1; + } + } + seaRelation.addElement("inner", w); + } + + seaRelation = new MultiPolygonRelation(seaRelation, wayMap); + relationMap.put(multiId, seaRelation); + seaRelation.processElements(); + } + + /** + * Specifies where an edge of the bounding box is hit. + */ + private static class EdgeHit implements Comparable<EdgeHit> + { + int edge; + double t; + + EdgeHit(int edge, double t) { + this.edge = edge; + this.t = t; + } + + public int compareTo(EdgeHit o) { + if (edge < o.edge) + return -1; + else if (edge > o.edge) + return +1; + else if (t < o.t) + return -1; + else if (t > o.t) + return 1; + else + return 0; + } + + Coord getPoint(Area a) { + log.debug("getPoint: ", this, a); + switch (edge) { + case 0: + return new Coord(a.getMinLat(), (int) (a.getMinLong() + t * (a.getMaxLong()-a.getMinLong()))); + + case 1: + return new Coord((int)(a.getMinLat() + t * (a.getMaxLat()-a.getMinLat())), a.getMaxLong()); + + case 2: + return new Coord(a.getMaxLat(), (int)(a.getMaxLong() - t * (a.getMaxLong()-a.getMinLong()))); + + case 3: + return new Coord((int)(a.getMaxLat() - t * (a.getMaxLat()-a.getMinLat())), a.getMinLong()); + + default: + throw new RuntimeException("illegal state"); + } + } + + public String toString() { + return "EdgeHit " + edge + "@" + t; + } + } + + private EdgeHit getEdgeHit(Area a, Coord p) + { + return getEdgeHit(a, p, 10); + } + + private EdgeHit getEdgeHit(Area a, Coord p, int tolerance) + { + int lat = p.getLatitude(); + int lon = p.getLongitude(); + int minLat = a.getMinLat(); + int maxLat = a.getMaxLat(); + int minLong = a.getMinLong(); + int maxLong = a.getMaxLong(); + + log.info(String.format("getEdgeHit: (%d %d) (%d %d %d %d)", lat, lon, minLat, minLong, maxLat, maxLong)); + if (lat <= minLat+tolerance) { + return new EdgeHit(0, ((double)(lon - minLong))/(maxLong-minLong)); + } + else if (lon >= maxLong-tolerance) { + return new EdgeHit(1, ((double)(lat - minLat))/(maxLat-minLat)); + } + else if (lat >= maxLat-tolerance) { + return new EdgeHit(2, ((double)(maxLong - lon))/(maxLong-minLong)); + } + else if (lon <= minLong+tolerance) { + return new EdgeHit(3, ((double)(maxLat - lat))/(maxLat-minLat)); + } + else + return null; + } + + private void concatenateWays(List<Way> ways) { + Map<Coord, Way> beginMap = new HashMap(); + + for (Way w : ways) { + if (!w.isClosed()) { + List<Coord> points = w.getPoints(); + beginMap.put(points.get(0), w); + } + } + + int merged = 1; + while (merged > 0) { + merged = 0; + for (Way w1 : ways) { + if (w1.isClosed()) continue; + + List<Coord> points1 = w1.getPoints(); + Way w2 = beginMap.get(points1.get(points1.size()-1)); + if (w2 != null) { + log.info("merging: ", ways.size(), w1.getId(), w2.getId()); + List<Coord> points2 = w2.getPoints(); + Way wm; + if (w1.getId() < (1L << 62)) { + wm = new Way((1L << 62) + nextFakeId++); + ways.remove(w1); + ways.add(wm); + wm.getPoints().addAll(points1); + beginMap.put(points1.get(0), wm); + } + else { + wm = w1; + } + wm.getPoints().addAll(points2); + ways.remove(w2); + beginMap.remove(points2.get(0)); + merged++; + break; + } + } + } + } } Index: src/uk/me/parabola/mkgmap/reader/osm/Element.java =================================================================== --- src/uk/me/parabola/mkgmap/reader/osm/Element.java (Revision 1124) +++ src/uk/me/parabola/mkgmap/reader/osm/Element.java (Arbeitskopie) @@ -86,6 +86,7 @@ * element. */ public void copyTags(Element other) { + if (other.tags != null) tags = other.tags.copy(); } @@ -97,4 +98,8 @@ if (this.name == null) this.name = name; } + + public Tags getTags() { + return tags; + } } Index: src/uk/me/parabola/mkgmap/reader/osm/Tags.java =================================================================== --- src/uk/me/parabola/mkgmap/reader/osm/Tags.java (Revision 1124) +++ src/uk/me/parabola/mkgmap/reader/osm/Tags.java (Arbeitskopie) @@ -16,7 +16,9 @@ */ package uk.me.parabola.mkgmap.reader.osm; +import java.util.HashMap; import java.util.Iterator; +import java.util.Map; /** * Store the tags that belong to an Element. @@ -120,7 +122,7 @@ } return null; } - + /** * Make a deep copy of this object. * @return A copy of this object. @@ -271,4 +273,22 @@ put(e.key, e.value); } } -} + + public void removeAll() { + for (int i = 0; i < capacity; i++){ + keys[i] = null; + values[i] = null; + } + size = 0; + } + + public Map<String, String> getKeyValues() { + Map<String, String> tagMap = new HashMap<String, String>(); + for (int i = 0; i < capacity; i++) + if (keys[i] != null && values[i] != null) + tagMap.put(keys[i], values[i]); + return tagMap; + } + + +} \ No newline at end of file Index: resources/styles/default/polygons =================================================================== --- resources/styles/default/polygons (Revision 1124) +++ resources/styles/default/polygons (Arbeitskopie) @@ -54,6 +54,7 @@ natural=mud [0x51 resolution 20] natural=scrub [0x4f resolution 20] natural=water [0x3c resolution 20] +natural=sea [0x32 resolution 10] natural=wood [0x50 resolution 18] place=village [0x03 resolution 18]

4 9

Commit: r1129: Fix disapearing multipolys if the first point happens to be the closest.
by svn commit 09 Aug '09

09 Aug '09

Version 1129 was commited by steve on 2009-08-09 23:14:56 +0100 (Sun, 09 Aug 2009) BRANCH: multipolygon Fix disapearing multipolys if the first point happens to be the closest. Modifed from patch sent by Christian, as -1 is technically a possible return value.

1 0

[PATCH v2] - styling for the power user
by Mark Burton 09 Aug '09

09 Aug '09

v2 now supports ~[0x??] syntax within name to specify highway shields to reduce the number of tags required, you can now specify all the values in the mkgmap:gtype tag like this example: mkgmap:gtype="0x20,19,,1,2" type = 0x20 minres = 19 maxres not specified roadclass = 1 roadspeed = 2 The one tag per value scheme is still supported (for the moment at least) -------------- Are you a heavy duty styler? If so, read on: I notice that quite a lot of postings on the list are from people who are having problems with complex style files. This little patch won't (directly) solve those problems but it does provide a useful capability that could be part of a solution for those people who have complex styling requirements and are willing to do some coding. What the patch does is allow elements (nodes, ways) to specify their garmin type (and a few other things) explicitly using these tags: mkgmap:gtype the element's type (integer constant) mkgmap:kind one of "node", "polyline", "polygon" (only polygon is used at this time to differentiate polygons from lines). mkgmap:minres the element's minimum resolution (needed to make element visible) mkgmap:maxres the element's maximum resolution (not required) mkgmap:roadclass the element's road class (needed for roads) mkgmap:roadspeed the element's road speed (needed for roads) If the mkgmap:gtype tag is present, the element will not be passed through the normal style file process at all. So how do these tags get added to the OSM data? Well, you could just add them with an editor but that would get boring pretty quickly so what I would expect to see is some kind of external filter program that reads the OSM file and outputs a new OSM file with the appropriate tags added. That filter program could be written in any language that has some XML processing support. Current issues to be sorted out are handling of the highway shields (not currently implemented) and also this feature is not compatible with the cycleway faking code. All feedback welcome. Cheers, Mark

9 16

[PATCH v3] - styling for the power user
by Mark Burton 09 Aug '09

09 Aug '09

v3 Now only has 1 tag (mkgmap:gtype) to specify everything. tag has the format 'kind,code,minres,maxres,roadclass,roadspeed' Where kind is 1 for node, 2 for polyline and 3 for polygon. kind, code and minres are all required to be present, the other values can be ommitted. Note, this format has changed since v2. ------------------------ v2 now supports ~[0x??] syntax within name to specify highway shields to reduce the number of tags required, you can now specify all the values in the mkgmap:gtype tag like this example: mkgmap:gtype="0x20,19,,1,2" type = 0x20 minres = 19 maxres not specified roadclass = 1 roadspeed = 2 The one tag per value scheme is still supported (for the moment at least) -------------- Are you a heavy duty styler? If so, read on: I notice that quite a lot of postings on the list are from people who are having problems with complex style files. This little patch won't (directly) solve those problems but it does provide a useful capability that could be part of a solution for those people who have complex styling requirements and are willing to do some coding. What the patch does is allow elements (nodes, ways) to specify their garmin type (and a few other things) explicitly using these tags: mkgmap:gtype the element's type (integer constant) mkgmap:kind one of "node", "polyline", "polygon" (only polygon is used at this time to differentiate polygons from lines). mkgmap:minres the element's minimum resolution (needed to make element visible) mkgmap:maxres the element's maximum resolution (not required) mkgmap:roadclass the element's road class (needed for roads) mkgmap:roadspeed the element's road speed (needed for roads) If the mkgmap:gtype tag is present, the element will not be passed through the normal style file process at all. So how do these tags get added to the OSM data? Well, you could just add them with an editor but that would get boring pretty quickly so what I would expect to see is some kind of external filter program that reads the OSM file and outputs a new OSM file with the appropriate tags added. That filter program could be written in any language that has some XML processing support. Current issues to be sorted out are handling of the highway shields (not currently implemented) and also this feature is not compatible with the cycleway faking code. All feedback welcome. Cheers, Mark

1 0

Commit: r1127: Move ant task to its original package.
by svn commit 09 Aug '09

09 Aug '09

Version 1127 was commited by steve on 2009-08-09 16:24:54 +0100 (Sun, 09 Aug 2009) Move ant task to its original package. Fix finding style off the class path for the ant task. Modified ant task to not require changes to the public interface of the argument reader. Document the long form of the -c option.

1 0

Commit: r1126: Fix unvalued options that are read from a file.
by svn commit 09 Aug '09

09 Aug '09

Version 1126 was commited by steve on 2009-08-09 15:19:40 +0100 (Sun, 09 Aug 2009) Fix unvalued options that are read from a file. I had kind of fixed that before, but not in the case where it was the last option.

1 0

Commit: r1125: Ant task for mkgmap.
by svn commit 09 Aug '09

09 Aug '09

Version 1125 was commited by steve on 2009-08-09 13:10:22 +0100 (Sun, 09 Aug 2009) Ant task for mkgmap. As submitted by Christian Gawron.

3 2

[PATCH] bzip2-support
by Christian Gawron 09 Aug '09

09 Aug '09

Here is a small patch which enables mkgmap to read bzip2-compressed files. It uses the org.apache.tools.bzip2.CBZip2InputStream class which is also used by the splitter. Best wishes Christian Index: src/uk/me/parabola/mkgmap/reader/MapperBasedMapDataSource.java =================================================================== --- src/uk/me/parabola/mkgmap/reader/MapperBasedMapDataSource.java (Revision 1115) +++ src/uk/me/parabola/mkgmap/reader/MapperBasedMapDataSource.java (Arbeitskopie) @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.List; import java.util.zip.GZIPInputStream; +import org.apache.tools.bzip2.CBZip2InputStream; import uk.me.parabola.imgfmt.app.Area; import uk.me.parabola.imgfmt.app.Coord; @@ -109,6 +110,14 @@ throw new FileNotFoundException( "Could not read as compressed file"); } } + else if (name.endsWith(".bz") || name.endsWith(".bz2")) { + try { + is.skip(2); + is = new CBZip2InputStream(is); + } catch (IOException e) { + throw new FileNotFoundException( "Could not read as compressed file"); + } + } return is; } /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * */ /* * This package is based on the work done by Keiron Liddle, Aftex Software * <keiron(a)aftexsw.com> to whom the Ant project is very grateful for his * great code. */ package org.apache.tools.bzip2; /** * A simple class the hold and calculate the CRC for sanity checking * of the data. * */ final class CRC { static final int crc32Table[] = { 0x00000000, 0x04c11db7, 0x09823b6e, 0x0d4326d9, 0x130476dc, 0x17c56b6b, 0x1a864db2, 0x1e475005, 0x2608edb8, 0x22c9f00f, 0x2f8ad6d6, 0x2b4bcb61, 0x350c9b64, 0x31cd86d3, 0x3c8ea00a, 0x384fbdbd, 0x4c11db70, 0x48d0c6c7, 0x4593e01e, 0x4152fda9, 0x5f15adac, 0x5bd4b01b, 0x569796c2, 0x52568b75, 0x6a1936c8, 0x6ed82b7f, 0x639b0da6, 0x675a1011, 0x791d4014, 0x7ddc5da3, 0x709f7b7a, 0x745e66cd, 0x9823b6e0, 0x9ce2ab57, 0x91a18d8e, 0x95609039, 0x8b27c03c, 0x8fe6dd8b, 0x82a5fb52, 0x8664e6e5, 0xbe2b5b58, 0xbaea46ef, 0xb7a96036, 0xb3687d81, 0xad2f2d84, 0xa9ee3033, 0xa4ad16ea, 0xa06c0b5d, 0xd4326d90, 0xd0f37027, 0xddb056fe, 0xd9714b49, 0xc7361b4c, 0xc3f706fb, 0xceb42022, 0xca753d95, 0xf23a8028, 0xf6fb9d9f, 0xfbb8bb46, 0xff79a6f1, 0xe13ef6f4, 0xe5ffeb43, 0xe8bccd9a, 0xec7dd02d, 0x34867077, 0x30476dc0, 0x3d044b19, 0x39c556ae, 0x278206ab, 0x23431b1c, 0x2e003dc5, 0x2ac12072, 0x128e9dcf, 0x164f8078, 0x1b0ca6a1, 0x1fcdbb16, 0x018aeb13, 0x054bf6a4, 0x0808d07d, 0x0cc9cdca, 0x7897ab07, 0x7c56b6b0, 0x71159069, 0x75d48dde, 0x6b93dddb, 0x6f52c06c, 0x6211e6b5, 0x66d0fb02, 0x5e9f46bf, 0x5a5e5b08, 0x571d7dd1, 0x53dc6066, 0x4d9b3063, 0x495a2dd4, 0x44190b0d, 0x40d816ba, 0xaca5c697, 0xa864db20, 0xa527fdf9, 0xa1e6e04e, 0xbfa1b04b, 0xbb60adfc, 0xb6238b25, 0xb2e29692, 0x8aad2b2f, 0x8e6c3698, 0x832f1041, 0x87ee0df6, 0x99a95df3, 0x9d684044, 0x902b669d, 0x94ea7b2a, 0xe0b41de7, 0xe4750050, 0xe9362689, 0xedf73b3e, 0xf3b06b3b, 0xf771768c, 0xfa325055, 0xfef34de2, 0xc6bcf05f, 0xc27dede8, 0xcf3ecb31, 0xcbffd686, 0xd5b88683, 0xd1799b34, 0xdc3abded, 0xd8fba05a, 0x690ce0ee, 0x6dcdfd59, 0x608edb80, 0x644fc637, 0x7a089632, 0x7ec98b85, 0x738aad5c, 0x774bb0eb, 0x4f040d56, 0x4bc510e1, 0x46863638, 0x42472b8f, 0x5c007b8a, 0x58c1663d, 0x558240e4, 0x51435d53, 0x251d3b9e, 0x21dc2629, 0x2c9f00f0, 0x285e1d47, 0x36194d42, 0x32d850f5, 0x3f9b762c, 0x3b5a6b9b, 0x0315d626, 0x07d4cb91, 0x0a97ed48, 0x0e56f0ff, 0x1011a0fa, 0x14d0bd4d, 0x19939b94, 0x1d528623, 0xf12f560e, 0xf5ee4bb9, 0xf8ad6d60, 0xfc6c70d7, 0xe22b20d2, 0xe6ea3d65, 0xeba91bbc, 0xef68060b, 0xd727bbb6, 0xd3e6a601, 0xdea580d8, 0xda649d6f, 0xc423cd6a, 0xc0e2d0dd, 0xcda1f604, 0xc960ebb3, 0xbd3e8d7e, 0xb9ff90c9, 0xb4bcb610, 0xb07daba7, 0xae3afba2, 0xaafbe615, 0xa7b8c0cc, 0xa379dd7b, 0x9b3660c6, 0x9ff77d71, 0x92b45ba8, 0x9675461f, 0x8832161a, 0x8cf30bad, 0x81b02d74, 0x857130c3, 0x5d8a9099, 0x594b8d2e, 0x5408abf7, 0x50c9b640, 0x4e8ee645, 0x4a4ffbf2, 0x470cdd2b, 0x43cdc09c, 0x7b827d21, 0x7f436096, 0x7200464f, 0x76c15bf8, 0x68860bfd, 0x6c47164a, 0x61043093, 0x65c52d24, 0x119b4be9, 0x155a565e, 0x18197087, 0x1cd86d30, 0x029f3d35, 0x065e2082, 0x0b1d065b, 0x0fdc1bec, 0x3793a651, 0x3352bbe6, 0x3e119d3f, 0x3ad08088, 0x2497d08d, 0x2056cd3a, 0x2d15ebe3, 0x29d4f654, 0xc5a92679, 0xc1683bce, 0xcc2b1d17, 0xc8ea00a0, 0xd6ad50a5, 0xd26c4d12, 0xdf2f6bcb, 0xdbee767c, 0xe3a1cbc1, 0xe760d676, 0xea23f0af, 0xeee2ed18, 0xf0a5bd1d, 0xf464a0aa, 0xf9278673, 0xfde69bc4, 0x89b8fd09, 0x8d79e0be, 0x803ac667, 0x84fbdbd0, 0x9abc8bd5, 0x9e7d9662, 0x933eb0bb, 0x97ffad0c, 0xafb010b1, 0xab710d06, 0xa6322bdf, 0xa2f33668, 0xbcb4666d, 0xb8757bda, 0xb5365d03, 0xb1f740b4 }; CRC() { initialiseCRC(); } void initialiseCRC() { globalCrc = 0xffffffff; } int getFinalCRC() { return ~globalCrc; } int getGlobalCRC() { return globalCrc; } void setGlobalCRC(int newCrc) { globalCrc = newCrc; } void updateCRC(int inCh) { int temp = (globalCrc >> 24) ^ inCh; if (temp < 0) { temp = 256 + temp; } globalCrc = (globalCrc << 8) ^ CRC.crc32Table[temp]; } void updateCRC(int inCh, int repeat) { int globalCrcShadow = this.globalCrc; while (repeat-- > 0) { int temp = (globalCrcShadow >> 24) ^ inCh; globalCrcShadow = (globalCrcShadow << 8) ^ crc32Table[(temp >= 0) ? temp : (temp + 256)]; } this.globalCrc = globalCrcShadow; } int globalCrc; } /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * */ /* * This package is based on the work done by Keiron Liddle, Aftex Software * <keiron(a)aftexsw.com> to whom the Ant project is very grateful for his * great code. */ package org.apache.tools.bzip2; /** * Base class for both the compress and decompress classes. * Holds common arrays, and static data. * <p> * This interface is public for historical purposes. * You should have no need to use it. * </p> */ public interface BZip2Constants { int baseBlockSize = 100000; int MAX_ALPHA_SIZE = 258; int MAX_CODE_LEN = 23; int RUNA = 0; int RUNB = 1; int N_GROUPS = 6; int G_SIZE = 50; int N_ITERS = 4; int MAX_SELECTORS = (2 + (900000 / G_SIZE)); int NUM_OVERSHOOT_BYTES = 20; /** * This array really shouldn't be here. * Again, for historical purposes it is. * * <p>FIXME: This array should be in a private or package private * location, since it could be modified by malicious code.</p> */ int[] rNums = { 619, 720, 127, 481, 931, 816, 813, 233, 566, 247, 985, 724, 205, 454, 863, 491, 741, 242, 949, 214, 733, 859, 335, 708, 621, 574, 73, 654, 730, 472, 419, 436, 278, 496, 867, 210, 399, 680, 480, 51, 878, 465, 811, 169, 869, 675, 611, 697, 867, 561, 862, 687, 507, 283, 482, 129, 807, 591, 733, 623, 150, 238, 59, 379, 684, 877, 625, 169, 643, 105, 170, 607, 520, 932, 727, 476, 693, 425, 174, 647, 73, 122, 335, 530, 442, 853, 695, 249, 445, 515, 909, 545, 703, 919, 874, 474, 882, 500, 594, 612, 641, 801, 220, 162, 819, 984, 589, 513, 495, 799, 161, 604, 958, 533, 221, 400, 386, 867, 600, 782, 382, 596, 414, 171, 516, 375, 682, 485, 911, 276, 98, 553, 163, 354, 666, 933, 424, 341, 533, 870, 227, 730, 475, 186, 263, 647, 537, 686, 600, 224, 469, 68, 770, 919, 190, 373, 294, 822, 808, 206, 184, 943, 795, 384, 383, 461, 404, 758, 839, 887, 715, 67, 618, 276, 204, 918, 873, 777, 604, 560, 951, 160, 578, 722, 79, 804, 96, 409, 713, 940, 652, 934, 970, 447, 318, 353, 859, 672, 112, 785, 645, 863, 803, 350, 139, 93, 354, 99, 820, 908, 609, 772, 154, 274, 580, 184, 79, 626, 630, 742, 653, 282, 762, 623, 680, 81, 927, 626, 789, 125, 411, 521, 938, 300, 821, 78, 343, 175, 128, 250, 170, 774, 972, 275, 999, 639, 495, 78, 352, 126, 857, 956, 358, 619, 580, 124, 737, 594, 701, 612, 669, 112, 134, 694, 363, 992, 809, 743, 168, 974, 944, 375, 748, 52, 600, 747, 642, 182, 862, 81, 344, 805, 988, 739, 511, 655, 814, 334, 249, 515, 897, 955, 664, 981, 649, 113, 974, 459, 893, 228, 433, 837, 553, 268, 926, 240, 102, 654, 459, 51, 686, 754, 806, 760, 493, 403, 415, 394, 687, 700, 946, 670, 656, 610, 738, 392, 760, 799, 887, 653, 978, 321, 576, 617, 626, 502, 894, 679, 243, 440, 680, 879, 194, 572, 640, 724, 926, 56, 204, 700, 707, 151, 457, 449, 797, 195, 791, 558, 945, 679, 297, 59, 87, 824, 713, 663, 412, 693, 342, 606, 134, 108, 571, 364, 631, 212, 174, 643, 304, 329, 343, 97, 430, 751, 497, 314, 983, 374, 822, 928, 140, 206, 73, 263, 980, 736, 876, 478, 430, 305, 170, 514, 364, 692, 829, 82, 855, 953, 676, 246, 369, 970, 294, 750, 807, 827, 150, 790, 288, 923, 804, 378, 215, 828, 592, 281, 565, 555, 710, 82, 896, 831, 547, 261, 524, 462, 293, 465, 502, 56, 661, 821, 976, 991, 658, 869, 905, 758, 745, 193, 768, 550, 608, 933, 378, 286, 215, 979, 792, 961, 61, 688, 793, 644, 986, 403, 106, 366, 905, 644, 372, 567, 466, 434, 645, 210, 389, 550, 919, 135, 780, 773, 635, 389, 707, 100, 626, 958, 165, 504, 920, 176, 193, 713, 857, 265, 203, 50, 668, 108, 645, 990, 626, 197, 510, 357, 358, 850, 858, 364, 936, 638 }; } /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * */ /* * This package is based on the work done by Keiron Liddle, Aftex Software * <keiron(a)aftexsw.com> to whom the Ant project is very grateful for his * great code. */ package org.apache.tools.bzip2; import java.io.IOException; import java.io.InputStream; /** * An input stream that decompresses from the BZip2 format (without the file * header chars) to be read as any other stream. * * <p>The decompression requires large amounts of memory. Thus you * should call the {@link #close() close()} method as soon as * possible, to force <tt>CBZip2InputStream</tt> to release the * allocated memory. See {@link CBZip2OutputStream * CBZip2OutputStream} for information about memory usage.</p> * * <p><tt>CBZip2InputStream</tt> reads bytes from the compressed * source stream via the single byte {@link java.io.InputStream#read() * read()} method exclusively. Thus you should consider to use a * buffered source stream.</p> * * <p>Instances of this class are not threadsafe.</p> */ public class CBZip2InputStream extends InputStream implements BZip2Constants { private static void reportCRCError() throws IOException { // The clean way would be to throw an exception. //throw new IOException("crc error"); // Just print a message, like the previous versions of this class did System.err.println("BZip2 CRC error"); } private void makeMaps() { final boolean[] inUse = this.data.inUse; final byte[] seqToUnseq = this.data.seqToUnseq; int nInUseShadow = 0; for (int i = 0; i < 256; i++) { if (inUse[i]) seqToUnseq[nInUseShadow++] = (byte) i; } this.nInUse = nInUseShadow; } /** * Index of the last char in the block, so the block size == last + 1. */ private int last; /** * Index in zptr[] of original string after sorting. */ private int origPtr; /** * always: in the range 0 .. 9. * The current block size is 100000 * this number. */ private int blockSize100k; private boolean blockRandomised; private int bsBuff; private int bsLive; private final CRC crc = new CRC(); private int nInUse; private InputStream in; private int currentChar = -1; private static final int EOF = 0; private static final int START_BLOCK_STATE = 1; private static final int RAND_PART_A_STATE = 2; private static final int RAND_PART_B_STATE = 3; private static final int RAND_PART_C_STATE = 4; private static final int NO_RAND_PART_A_STATE = 5; private static final int NO_RAND_PART_B_STATE = 6; private static final int NO_RAND_PART_C_STATE = 7; private int currentState = START_BLOCK_STATE; private int storedBlockCRC, storedCombinedCRC; private int computedBlockCRC, computedCombinedCRC; // Variables used by setup* methods exclusively private int su_count; private int su_ch2; private int su_chPrev; private int su_i2; private int su_j2; private int su_rNToGo; private int su_rTPos; private int su_tPos; private char su_z; /** * All memory intensive stuff. * This field is initialized by initBlock(). */ private CBZip2InputStream.Data data; /** * Constructs a new CBZip2InputStream which decompresses bytes read from * the specified stream. * * <p>Although BZip2 headers are marked with the magic * <tt>"Bz"</tt> this constructor expects the next byte in the * stream to be the first one after the magic. Thus callers have * to skip the first two bytes. Otherwise this constructor will * throw an exception. </p> * * @throws IOException * if the stream content is malformed or an I/O error occurs. * @throws NullPointerException * if <tt>in == null</tt> */ public CBZip2InputStream(final InputStream in) throws IOException { super(); this.in = in; init(); } public int read() throws IOException { if (this.in != null) { return read0(); } else { throw new IOException("stream closed"); } } public int read(final byte[] dest, final int offs, final int len) throws IOException { if (offs < 0) { throw new IndexOutOfBoundsException("offs(" + offs + ") < 0."); } if (len < 0) { throw new IndexOutOfBoundsException("len(" + len + ") < 0."); } if (offs + len > dest.length) { throw new IndexOutOfBoundsException("offs(" + offs + ") + len(" + len + ") > dest.length(" + dest.length + ")."); } if (this.in == null) { throw new IOException("stream closed"); } final int hi = offs + len; int destOffs = offs; for (int b; (destOffs < hi) && ((b = read0()) >= 0);) { dest[destOffs++] = (byte) b; } return (destOffs == offs) ? -1 : (destOffs - offs); } private int read0() throws IOException { final int retChar = this.currentChar; switch (this.currentState) { case EOF: return -1; case START_BLOCK_STATE: throw new IllegalStateException(); case RAND_PART_A_STATE: throw new IllegalStateException(); case RAND_PART_B_STATE: setupRandPartB(); break; case RAND_PART_C_STATE: setupRandPartC(); break; case NO_RAND_PART_A_STATE: throw new IllegalStateException(); case NO_RAND_PART_B_STATE: setupNoRandPartB(); break; case NO_RAND_PART_C_STATE: setupNoRandPartC(); break; default: throw new IllegalStateException(); } return retChar; } private void init() throws IOException { int magic2 = this.in.read(); if (magic2 != 'h') { throw new IOException("Stream is not BZip2 formatted: expected 'h'" + " as first byte but got '" + (char) magic2 + "'"); } int blockSize = this.in.read(); if ((blockSize < '1') || (blockSize > '9')) { throw new IOException("Stream is not BZip2 formatted: illegal " + "blocksize " + (char) blockSize); } this.blockSize100k = blockSize - '0'; initBlock(); setupBlock(); } private void initBlock() throws IOException { char magic0 = bsGetUByte(); char magic1 = bsGetUByte(); char magic2 = bsGetUByte(); char magic3 = bsGetUByte(); char magic4 = bsGetUByte(); char magic5 = bsGetUByte(); if (magic0 == 0x17 && magic1 == 0x72 && magic2 == 0x45 && magic3 == 0x38 && magic4 == 0x50 && magic5 == 0x90) { complete(); // end of file } else if (magic0 != 0x31 || // '1' magic1 != 0x41 || // ')' magic2 != 0x59 || // 'Y' magic3 != 0x26 || // '&' magic4 != 0x53 || // 'S' magic5 != 0x59 // 'Y' ) { this.currentState = EOF; throw new IOException("bad block header"); } else { this.storedBlockCRC = bsGetInt(); this.blockRandomised = bsR(1) == 1; /** * Allocate data here instead in constructor, so we do not * allocate it if the input file is empty. */ if (this.data == null) { this.data = new Data(this.blockSize100k); } // currBlockNo++; getAndMoveToFrontDecode(); this.crc.initialiseCRC(); this.currentState = START_BLOCK_STATE; } } private void endBlock() throws IOException { this.computedBlockCRC = this.crc.getFinalCRC(); // A bad CRC is considered a fatal error. if (this.storedBlockCRC != this.computedBlockCRC) { // make next blocks readable without error // (repair feature, not yet documented, not tested) this.computedCombinedCRC = (this.storedCombinedCRC << 1) | (this.storedCombinedCRC >>> 31); this.computedCombinedCRC ^= this.storedBlockCRC; reportCRCError(); } this.computedCombinedCRC = (this.computedCombinedCRC << 1) | (this.computedCombinedCRC >>> 31); this.computedCombinedCRC ^= this.computedBlockCRC; } private void complete() throws IOException { this.storedCombinedCRC = bsGetInt(); this.currentState = EOF; this.data = null; if (this.storedCombinedCRC != this.computedCombinedCRC) { reportCRCError(); } } public void close() throws IOException { InputStream inShadow = this.in; if (inShadow != null) { try { if (inShadow != System.in) { inShadow.close(); } } finally { this.data = null; this.in = null; } } } private int bsR(final int n) throws IOException { int bsLiveShadow = this.bsLive; int bsBuffShadow = this.bsBuff; if (bsLiveShadow < n) { final InputStream inShadow = this.in; do { int thech = inShadow.read(); if (thech < 0) { throw new IOException("unexpected end of stream"); } bsBuffShadow = (bsBuffShadow << 8) | thech; bsLiveShadow += 8; } while (bsLiveShadow < n); this.bsBuff = bsBuffShadow; } this.bsLive = bsLiveShadow - n; return (bsBuffShadow >> (bsLiveShadow - n)) & ((1 << n) - 1); } private boolean bsGetBit() throws IOException { int bsLiveShadow = this.bsLive; int bsBuffShadow = this.bsBuff; if (bsLiveShadow < 1) { int thech = this.in.read(); if (thech < 0) { throw new IOException("unexpected end of stream"); } bsBuffShadow = (bsBuffShadow << 8) | thech; bsLiveShadow += 8; this.bsBuff = bsBuffShadow; } this.bsLive = bsLiveShadow - 1; return ((bsBuffShadow >> (bsLiveShadow - 1)) & 1) != 0; } private char bsGetUByte() throws IOException { return (char) bsR(8); } private int bsGetInt() throws IOException { return (((((bsR(8) << 8) | bsR(8)) << 8) | bsR(8)) << 8) | bsR(8); } /** * Called by createHuffmanDecodingTables() exclusively. */ private static void hbCreateDecodeTables(final int[] limit, final int[] base, final int[] perm, final char[] length, final int minLen, final int maxLen, final int alphaSize) { for (int i = minLen, pp = 0; i <= maxLen; i++) { for (int j = 0; j < alphaSize; j++) { if (length[j] == i) { perm[pp++] = j; } } } for (int i = MAX_CODE_LEN; --i > 0;) { base[i] = 0; limit[i] = 0; } for (int i = 0; i < alphaSize; i++) { base[length[i] + 1]++; } for (int i = 1, b = base[0]; i < MAX_CODE_LEN; i++) { b += base[i]; base[i] = b; } for (int i = minLen, vec = 0, b = base[i]; i <= maxLen; i++) { final int nb = base[i + 1]; vec += nb - b; b = nb; limit[i] = vec - 1; vec <<= 1; } for (int i = minLen + 1; i <= maxLen; i++) { base[i] = ((limit[i - 1] + 1) << 1) - base[i]; } } private void recvDecodingTables() throws IOException { final Data dataShadow = this.data; final boolean[] inUse = dataShadow.inUse; final byte[] pos = dataShadow.recvDecodingTables_pos; final byte[] selector = dataShadow.selector; final byte[] selectorMtf = dataShadow.selectorMtf; int inUse16 = 0; /* Receive the mapping table */ for (int i = 0; i < 16; i++) { if (bsGetBit()) { inUse16 |= 1 << i; } } for (int i = 256; --i >= 0;) { inUse[i] = false; } for (int i = 0; i < 16; i++) { if ((inUse16 & (1 << i)) != 0) { final int i16 = i << 4; for (int j = 0; j < 16; j++) { if (bsGetBit()) { inUse[i16 + j] = true; } } } } makeMaps(); final int alphaSize = this.nInUse + 2; /* Now the selectors */ final int nGroups = bsR(3); final int nSelectors = bsR(15); for (int i = 0; i < nSelectors; i++) { int j = 0; while (bsGetBit()) { j++; } selectorMtf[i] = (byte) j; } /* Undo the MTF values for the selectors. */ for (int v = nGroups; --v >= 0;) { pos[v] = (byte) v; } for (int i = 0; i < nSelectors; i++) { int v = selectorMtf[i] & 0xff; final byte tmp = pos[v]; while (v > 0) { // nearly all times v is zero, 4 in most other cases pos[v] = pos[v - 1]; v--; } pos[0] = tmp; selector[i] = tmp; } final char[][] len = dataShadow.temp_charArray2d; /* Now the coding tables */ for (int t = 0; t < nGroups; t++) { int curr = bsR(5); final char[] len_t = len[t]; for (int i = 0; i < alphaSize; i++) { while (bsGetBit()) { curr += bsGetBit() ? -1 : 1; } len_t[i] = (char) curr; } } // finally create the Huffman tables createHuffmanDecodingTables(alphaSize, nGroups); } /** * Called by recvDecodingTables() exclusively. */ private void createHuffmanDecodingTables(final int alphaSize, final int nGroups) { final Data dataShadow = this.data; final char[][] len = dataShadow.temp_charArray2d; final int[] minLens = dataShadow.minLens; final int[][] limit = dataShadow.limit; final int[][] base = dataShadow.base; final int[][] perm = dataShadow.perm; for (int t = 0; t < nGroups; t++) { int minLen = 32; int maxLen = 0; final char[] len_t = len[t]; for (int i = alphaSize; --i >= 0;) { final char lent = len_t[i]; if (lent > maxLen) { maxLen = lent; } if (lent < minLen) { minLen = lent; } } hbCreateDecodeTables(limit[t], base[t], perm[t], len[t], minLen, maxLen, alphaSize); minLens[t] = minLen; } } private void getAndMoveToFrontDecode() throws IOException { this.origPtr = bsR(24); recvDecodingTables(); final InputStream inShadow = this.in; final Data dataShadow = this.data; final byte[] ll8 = dataShadow.ll8; final int[] unzftab = dataShadow.unzftab; final byte[] selector = dataShadow.selector; final byte[] seqToUnseq = dataShadow.seqToUnseq; final char[] yy = dataShadow.getAndMoveToFrontDecode_yy; final int[] minLens = dataShadow.minLens; final int[][] limit = dataShadow.limit; final int[][] base = dataShadow.base; final int[][] perm = dataShadow.perm; final int limitLast = this.blockSize100k * 100000; /* Setting up the unzftab entries here is not strictly necessary, but it does save having to do it later in a separate pass, and so saves a block's worth of cache misses. */ for (int i = 256; --i >= 0;) { yy[i] = (char) i; unzftab[i] = 0; } int groupNo = 0; int groupPos = G_SIZE - 1; final int eob = this.nInUse + 1; int nextSym = getAndMoveToFrontDecode0(0); int bsBuffShadow = this.bsBuff; int bsLiveShadow = this.bsLive; int lastShadow = -1; int zt = selector[groupNo] & 0xff; int[] base_zt = base[zt]; int[] limit_zt = limit[zt]; int[] perm_zt = perm[zt]; int minLens_zt = minLens[zt]; while (nextSym != eob) { if ((nextSym == RUNA) || (nextSym == RUNB)) { int s = -1; for (int n = 1; true; n <<= 1) { if (nextSym == RUNA) { s += n; } else if (nextSym == RUNB) { s += n << 1; } else { break; } if (groupPos == 0) { groupPos = G_SIZE - 1; zt = selector[++groupNo] & 0xff; base_zt = base[zt]; limit_zt = limit[zt]; perm_zt = perm[zt]; minLens_zt = minLens[zt]; } else { groupPos--; } int zn = minLens_zt; // Inlined: // int zvec = bsR(zn); while (bsLiveShadow < zn) { final int thech = inShadow.read(); if (thech >= 0) { bsBuffShadow = (bsBuffShadow << 8) | thech; bsLiveShadow += 8; continue; } else { throw new IOException("unexpected end of stream"); } } int zvec = (bsBuffShadow >> (bsLiveShadow - zn)) & ((1 << zn) - 1); bsLiveShadow -= zn; while (zvec > limit_zt[zn]) { zn++; while (bsLiveShadow < 1) { final int thech = inShadow.read(); if (thech >= 0) { bsBuffShadow = (bsBuffShadow << 8) | thech; bsLiveShadow += 8; continue; } else { throw new IOException("unexpected end of stream"); } } bsLiveShadow--; zvec = (zvec << 1) | ((bsBuffShadow >> bsLiveShadow) & 1); } nextSym = perm_zt[zvec - base_zt[zn]]; } final byte ch = seqToUnseq[yy[0]]; unzftab[ch & 0xff] += s + 1; while (s-- >= 0) { ll8[++lastShadow] = ch; } if (lastShadow >= limitLast) { throw new IOException("block overrun"); } } else { if (++lastShadow >= limitLast) { throw new IOException("block overrun"); } final char tmp = yy[nextSym - 1]; unzftab[seqToUnseq[tmp] & 0xff]++; ll8[lastShadow] = seqToUnseq[tmp]; /* This loop is hammered during decompression, hence avoid native method call overhead of System.arraycopy for very small ranges to copy. */ if (nextSym <= 16) { for (int j = nextSym - 1; j > 0;) { yy[j] = yy[--j]; } } else { System.arraycopy(yy, 0, yy, 1, nextSym - 1); } yy[0] = tmp; if (groupPos == 0) { groupPos = G_SIZE - 1; zt = selector[++groupNo] & 0xff; base_zt = base[zt]; limit_zt = limit[zt]; perm_zt = perm[zt]; minLens_zt = minLens[zt]; } else { groupPos--; } int zn = minLens_zt; // Inlined: // int zvec = bsR(zn); while (bsLiveShadow < zn) { final int thech = inShadow.read(); if (thech >= 0) { bsBuffShadow = (bsBuffShadow << 8) | thech; bsLiveShadow += 8; continue; } else { throw new IOException("unexpected end of stream"); } } int zvec = (bsBuffShadow >> (bsLiveShadow - zn)) & ((1 << zn) - 1); bsLiveShadow -= zn; while (zvec > limit_zt[zn]) { zn++; while (bsLiveShadow < 1) { final int thech = inShadow.read(); if (thech >= 0) { bsBuffShadow = (bsBuffShadow << 8) | thech; bsLiveShadow += 8; continue; } else { throw new IOException("unexpected end of stream"); } } bsLiveShadow--; zvec = (zvec << 1) | ((bsBuffShadow >> bsLiveShadow) & 1); } nextSym = perm_zt[zvec - base_zt[zn]]; } } this.last = lastShadow; this.bsLive = bsLiveShadow; this.bsBuff = bsBuffShadow; } private int getAndMoveToFrontDecode0(final int groupNo) throws IOException { final InputStream inShadow = this.in; final Data dataShadow = this.data; final int zt = dataShadow.selector[groupNo] & 0xff; final int[] limit_zt = dataShadow.limit[zt]; int zn = dataShadow.minLens[zt]; int zvec = bsR(zn); int bsLiveShadow = this.bsLive; int bsBuffShadow = this.bsBuff; while (zvec > limit_zt[zn]) { zn++; while (bsLiveShadow < 1) { final int thech = inShadow.read(); if (thech >= 0) { bsBuffShadow = (bsBuffShadow << 8) | thech; bsLiveShadow += 8; continue; } else { throw new IOException("unexpected end of stream"); } } bsLiveShadow--; zvec = (zvec << 1) | ((bsBuffShadow >> bsLiveShadow) & 1); } this.bsLive = bsLiveShadow; this.bsBuff = bsBuffShadow; return dataShadow.perm[zt][zvec - dataShadow.base[zt][zn]]; } private void setupBlock() throws IOException { if (this.data == null) { return; } final int[] cftab = this.data.cftab; final int[] tt = this.data.initTT(this.last + 1); final byte[] ll8 = this.data.ll8; cftab[0] = 0; System.arraycopy(this.data.unzftab, 0, cftab, 1, 256); for (int i = 1, c = cftab[0]; i <= 256; i++) { c += cftab[i]; cftab[i] = c; } for (int i = 0, lastShadow = this.last; i <= lastShadow; i++) { tt[cftab[ll8[i] & 0xff]++] = i; } if ((this.origPtr < 0) || (this.origPtr >= tt.length)) { throw new IOException("stream corrupted"); } this.su_tPos = tt[this.origPtr]; this.su_count = 0; this.su_i2 = 0; this.su_ch2 = 256; /* not a char and not EOF */ if (this.blockRandomised) { this.su_rNToGo = 0; this.su_rTPos = 0; setupRandPartA(); } else { setupNoRandPartA(); } } private void setupRandPartA() throws IOException { if (this.su_i2 <= this.last) { this.su_chPrev = this.su_ch2; int su_ch2Shadow = this.data.ll8[this.su_tPos] & 0xff; this.su_tPos = this.data.tt[this.su_tPos]; if (this.su_rNToGo == 0) { this.su_rNToGo = BZip2Constants.rNums[this.su_rTPos] - 1; if (++this.su_rTPos == 512) { this.su_rTPos = 0; } } else { this.su_rNToGo--; } this.su_ch2 = su_ch2Shadow ^= (this.su_rNToGo == 1) ? 1 : 0; this.su_i2++; this.currentChar = su_ch2Shadow; this.currentState = RAND_PART_B_STATE; this.crc.updateCRC(su_ch2Shadow); } else { endBlock(); initBlock(); setupBlock(); } } private void setupNoRandPartA() throws IOException { if (this.su_i2 <= this.last) { this.su_chPrev = this.su_ch2; int su_ch2Shadow = this.data.ll8[this.su_tPos] & 0xff; this.su_ch2 = su_ch2Shadow; this.su_tPos = this.data.tt[this.su_tPos]; this.su_i2++; this.currentChar = su_ch2Shadow; this.currentState = NO_RAND_PART_B_STATE; this.crc.updateCRC(su_ch2Shadow); } else { this.currentState = NO_RAND_PART_A_STATE; endBlock(); initBlock(); setupBlock(); } } private void setupRandPartB() throws IOException { if (this.su_ch2 != this.su_chPrev) { this.currentState = RAND_PART_A_STATE; this.su_count = 1; setupRandPartA(); } else if (++this.su_count >= 4) { this.su_z = (char) (this.data.ll8[this.su_tPos] & 0xff); this.su_tPos = this.data.tt[this.su_tPos]; if (this.su_rNToGo == 0) { this.su_rNToGo = BZip2Constants.rNums[this.su_rTPos] - 1; if (++this.su_rTPos == 512) { this.su_rTPos = 0; } } else { this.su_rNToGo--; } this.su_j2 = 0; this.currentState = RAND_PART_C_STATE; if (this.su_rNToGo == 1) { this.su_z ^= 1; } setupRandPartC(); } else { this.currentState = RAND_PART_A_STATE; setupRandPartA(); } } private void setupRandPartC() throws IOException { if (this.su_j2 < this.su_z) { this.currentChar = this.su_ch2; this.crc.updateCRC(this.su_ch2); this.su_j2++; } else { this.currentState = RAND_PART_A_STATE; this.su_i2++; this.su_count = 0; setupRandPartA(); } } private void setupNoRandPartB() throws IOException { if (this.su_ch2 != this.su_chPrev) { this.su_count = 1; setupNoRandPartA(); } else if (++this.su_count >= 4) { this.su_z = (char) (this.data.ll8[this.su_tPos] & 0xff); this.su_tPos = this.data.tt[this.su_tPos]; this.su_j2 = 0; setupNoRandPartC(); } else { setupNoRandPartA(); } } private void setupNoRandPartC() throws IOException { if (this.su_j2 < this.su_z) { int su_ch2Shadow = this.su_ch2; this.currentChar = su_ch2Shadow; this.crc.updateCRC(su_ch2Shadow); this.su_j2++; this.currentState = NO_RAND_PART_C_STATE; } else { this.su_i2++; this.su_count = 0; setupNoRandPartA(); } } private static final class Data extends Object { // (with blockSize 900k) final boolean[] inUse = new boolean[256]; // 256 byte final byte[] seqToUnseq = new byte[256]; // 256 byte final byte[] selector = new byte[MAX_SELECTORS]; // 18002 byte final byte[] selectorMtf = new byte[MAX_SELECTORS]; // 18002 byte /** * Freq table collected to save a pass over the data during * decompression. */ final int[] unzftab = new int[256]; // 1024 byte final int[][] limit = new int[N_GROUPS][MAX_ALPHA_SIZE]; // 6192 byte final int[][] base = new int[N_GROUPS][MAX_ALPHA_SIZE]; // 6192 byte final int[][] perm = new int[N_GROUPS][MAX_ALPHA_SIZE]; // 6192 byte final int[] minLens = new int[N_GROUPS]; // 24 byte final int[] cftab = new int[257]; // 1028 byte final char[] getAndMoveToFrontDecode_yy = new char[256]; // 512 byte final char[][] temp_charArray2d = new char[N_GROUPS][MAX_ALPHA_SIZE]; // 3096 byte final byte[] recvDecodingTables_pos = new byte[N_GROUPS]; // 6 byte //--------------- // 60798 byte int[] tt; // 3600000 byte byte[] ll8; // 900000 byte //--------------- // 4560782 byte //=============== Data(int blockSize100k) { super(); this.ll8 = new byte[blockSize100k * BZip2Constants.baseBlockSize]; } /** * Initializes the {@link #tt} array. * * This method is called when the required length of the array * is known. I don't initialize it at construction time to * avoid unneccessary memory allocation when compressing small * files. */ final int[] initTT(int length) { int[] ttShadow = this.tt; // tt.length should always be >= length, but theoretically // it can happen, if the compressor mixed small and large // blocks. Normally only the last block will be smaller // than others. if ((ttShadow == null) || (ttShadow.length < length)) { this.tt = ttShadow = new int[length]; } return ttShadow; } } }

1 0

splitter build fails on mac
by Greg Troxel 08 Aug '09

08 Aug '09

(This is also splitter-dev I think.) I am trying to build the splitter on a mac (10.5.7 with java 1.6.0_13, 64-bit server VM), and failing. I have successfully built mkgmap and run it, so my java setup is mostly ok. Buildfile: build.xml prepare: compile: [javac] Compiling 17 source files to /Users/gdt/SOFTWARE/OPENSTREETMAP/splitter/build/classes [javac] /Users/gdt/SOFTWARE/OPENSTREETMAP/splitter/src/uk/me/parabola/splitter/AreaList.java:73: cannot find symbol [javac] symbol : variable ROOT [javac] location: class java.util.Locale [javac] pw.format(Locale.ROOT, "%d: %d,%d to %d,%d\n", [javac] ^ [javac] /Users/gdt/SOFTWARE/OPENSTREETMAP/splitter/src/uk/me/parabola/splitter/AreaList.java:77: cannot find symbol [javac] symbol : variable ROOT [javac] location: class java.util.Locale [javac] pw.format(Locale.ROOT, "# : %f,%f to %f,%f\n", [javac] ^ [javac] /Users/gdt/SOFTWARE/OPENSTREETMAP/splitter/src/uk/me/parabola/splitter/AreaList.java:110: cannot find symbol [javac] symbol : method isEmpty() [javac] location: class java.lang.String [javac] if (line.isEmpty() || line.charAt(0) == '#') [javac] ^ [javac] /Users/gdt/SOFTWARE/OPENSTREETMAP/splitter/src/uk/me/parabola/splitter/SubArea.java:93: cannot find symbol [javac] symbol : variable ROOT [javac] location: class java.util.Locale [javac] String filename = new Formatter().format(Locale.ROOT, "%08d.osm.gz", mapid).toString(); [javac] ^ [javac] 4 errors BUILD FAILED /Users/gdt/SOFTWARE/OPENSTREETMAP/splitter/build.xml:49: Compile failed; see the compiler error output for details. Total time: 0 seconds

4 7

Append Name with general rules via style-file
by Felix Hartmann 08 Aug '09

08 Aug '09

Some of you have already made patches for the "name" so maybe you know whether the following would be feasible. Currently add or set name will delete everything and put in the new text. the start-with used by Thilo's patches works AFAIK (I never tried it) also only following highway=* [0x?? level ?] What I would like is to have rules like: highway=* {name '${name} ${name1} (${ref})' | '${name1} ${ref}' | '${name1} ${name}' | '${name1}' | '${name} (${ref})' | '${ref}' | '${name}'} Where name1 is set in relations file. Currently name1 will simply replace name. I don't know why. Or highway=* & bridge=yes { name '${name} (${ref}) bridge' | '${ref} bridge' | '${name} bridge' } ....... (I would like to have a list of around 200 checks here, adding tracktype and similar to the name - so each rule should simply add itself on the name, currently not even having a single rule like the above works, so if one rule would work I would still excpect it to somehow fail or give surprising results as the rules are not run in order, but better a bit working than nothing.....) Which also currently does not work.

7 15