Rev 176 | Blame | Compare with Previous | Last modification | View Log | RSS feed
/*
* Copyright (c) 2009.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 3 as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*/
package uk.me.parabola.splitter;
import crosby.binary.file.BlockInputStream;
import org.xmlpull.v1.XmlPullParserException;
import uk.me.parabola.splitter.args.ParamParser;
import uk.me.parabola.splitter.args.SplitterParams;
import uk.me.parabola.splitter.geo.City;
import uk.me.parabola.splitter.geo.CityFinder;
import uk.me.parabola.splitter.geo.CityLoader;
import uk.me.parabola.splitter.geo.DefaultCityFinder;
import uk.me.parabola.splitter.geo.DummyCityFinder;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.io.Reader;
import java.nio.charset.Charset;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* Splitter for OSM files with the purpose of providing input files for mkgmap.
* <p/>
* The input file is split so that no piece has more than a given number of nodes in it.
*
* @author Steve Ratcliffe
*/
public class Main {
private static final String DEFAULT_DIR = ".";
// We can only process a maximum of 255 areas at a time because we
// compress an area ID into 8 bits to save memory (and 0 is reserved)
private int maxAreasPerPass;
// A list of the OSM files to parse.
private List<String> filenames;
// The description to write into the template.args file.
private String description;
// The starting map ID.
private int mapId;
// The amount in map units that tiles overlap (note that the final img's will not overlap
// but the input files do).
private int overlapAmount;
// The max number of nodes that will appear in a single file.
private int maxNodes;
// The maximum resolution of the map to be produced by mkgmap. This is a value in the range
// 0-24. Higher numbers mean higher detail. The resolution determines how the tiles must
// be aligned. Eg a resolution of 13 means the tiles need to have their edges aligned to
// multiples of 2 ^ (24 - 13) = 2048 map units, and their widths and heights must be a multiple
// of 2 * 2 ^ (24 - 13) = 4096 units. The tile widths and height multiples are double the tile
// alignment because the center point of the tile is stored, and that must be aligned the
// same as the tile edges are.
private int resolution;
// Whether or not to trim tiles of any empty space around their edges.
private boolean trim;
// This gets set if no osm file is supplied as a parameter and the cache is empty.
private boolean useStdIn;
// Set if there is a previous area file given on the command line.
private AreaList areaList;
// Whether or not the source OSM file(s) contain strictly nodes first, then ways, then rels,
// or they're all mixed up. Running with mixed enabled takes longer.
private boolean mixed;
// The path where the results are written out to.
private File fileOutputDir;
// A GeoNames file to use for naming the tiles.
private String geoNamesFile;
// How often (in seconds) to provide JVM status information. Zero = no information.
private int statusFreq;
// Whether to use the density map. Disabling this (not recommended) causes the splitter to
// revert to using legacy mode which takes MUCH more memory during phase one.
private boolean densityMap;
private String kmlOutputFile;
// The maximum number of threads the splitter should use.
private int maxThreads;
// The output type
private boolean pbfOutput;
public static void main(String[] args) {
Main m = new Main();
m.start(args);
}
private void start(String[] args) {
readArgs(args);
if (statusFreq > 0) {
JVMHealthMonitor.start(statusFreq);
}
long start = System.currentTimeMillis();
System.out.println("Time started: " + new Date());
try {
split();
} catch (IOException e) {
System.err.println("Error opening or reading file " + e);
e.printStackTrace();
} catch (XmlPullParserException e) {
System.err.println("Error parsing xml from file " + e);
e.printStackTrace();
}
System.out.println("Time finished: " + new Date());
System.out.println("Total time taken: " + (System.currentTimeMillis() - start) / 1000 + 's');
}
private void split() throws IOException, XmlPullParserException {
File outputDir = fileOutputDir;
if (!outputDir.exists()) {
System.out.println("Output directory not found. Creating directory '" + fileOutputDir + "'");
if (!outputDir.mkdirs()) {
System.err.println("Unable to create output directory! Using default directory instead");
fileOutputDir = new File(DEFAULT_DIR);
}
} else if (!outputDir.isDirectory()) {
System.err.println("The --output-dir parameter must specify a directory. The --output-dir parameter is being ignored, writing to default directory instead.");
fileOutputDir = new File(DEFAULT_DIR);
}
if (filenames.isEmpty()) {
if (areaList == null) {
throw new IllegalArgumentException("No .osm files were supplied so at least one of --cache or --split-file must be specified");
} else {
int areaCount = areaList.getAreas().size();
int passes = getAreasPerPass(areaCount);
if (passes > 1) {
throw new IllegalArgumentException("No .osm files or --cache parameter were supplied, but stdin cannot be used because " + passes
+ " passes are required to write out the areas. Either provide --cache or increase --max-areas to match the number of areas (" + areaCount + ')');
}
useStdIn = true;
}
}
if (areaList == null) {
int alignment = 1 << (24 - resolution);
System.out.println("Map is being split for resolution " + resolution + ':');
System.out.println(" - area boundaries are aligned to 0x" + Integer.toHexString(alignment) + " map units");
System.out.println(" - areas are multiples of 0x" + Integer.toHexString(alignment * 2) + " map units wide and high");
areaList = calculateAreas();
for (Area area : areaList.getAreas()) {
area.setMapId(mapId++);
}
nameAreas();
areaList.write(new File(fileOutputDir, "areas.list").getPath());
} else {
nameAreas();
}
List<Area> areas = areaList.getAreas();
System.out.println(areas.size() + " areas:");
for (Area area : areas) {
System.out.print("Area " + area.getMapId() + " covers " + area.toHexString());
if (area.getName() != null)
System.out.print(' ' + area.getName());
System.out.println();
}
if (kmlOutputFile != null) {
File out = new File(kmlOutputFile);
if (!out.isAbsolute())
kmlOutputFile = new File(fileOutputDir, kmlOutputFile).getPath();
System.out.println("Writing KML file to " + kmlOutputFile);
areaList.writeKml(kmlOutputFile);
}
writeAreas(areas);
writeArgsFile(areas);
}
private int getAreasPerPass(int areaCount) {
return (int) Math.ceil((double) areaCount / (double) maxAreasPerPass);
}
/**
* Deal with the command line arguments.
*/
private void readArgs(String[] args) {
ParamParser parser = new ParamParser();
SplitterParams params = parser.parse(SplitterParams.class, args);
if (!parser.getErrors().isEmpty()) {
System.out.println();
System.out.println("Invalid parameter(s):");
for (String error : parser.getErrors()) {
System.out.println(" " + error);
}
System.out.println();
parser.displayUsage();
System.exit(-1);
}
for (Map.Entry<String, Object> entry : parser.getConvertedParams().entrySet()) {
String name = entry.getKey();
Object value = entry.getValue();
System.out.println(name + '=' + (value == null ? "" : value));
}
mapId = params.getMapid();
overlapAmount = params.getOverlap();
maxNodes = params.getMaxNodes();
description = params.getDescription();
geoNamesFile = params.getGeonamesFile();
resolution = params.getResolution();
trim = !params.isNoTrim();
String output = params.getOutput();
// Remove warning and make the default pbf after a while.
if (output.equals("unset")) {
System.err.println("\n\n**** WARNING: the default output type has changed to pbf, use --output=xml for .osm.gz files\n");
output = "pbf";
}
if(!output.equals("xml") && !output.equals("pbf")) {
System.err.println("The --output parameter must be either xml or pbf. Resetting to xml.");
}
pbfOutput = "pbf".equals(output);
if (resolution < 1 || resolution > 24) {
System.err.println("The --resolution parameter must be a value between 1 and 24. Resetting to 13.");
resolution = 13;
}
mixed = params.isMixed();
statusFreq = params.getStatusFreq();
String outputDir = params.getOutputDir();
fileOutputDir = new File(outputDir == null? DEFAULT_DIR: outputDir);
maxAreasPerPass = params.getMaxAreas();
if (maxAreasPerPass < 1 || maxAreasPerPass > 255) {
System.err.println("The --max-areas parameter must be a value between 1 and 255. Resetting to 255.");
maxAreasPerPass = 255;
}
kmlOutputFile = params.getWriteKml();
densityMap = !params.isLegacyMode();
if (!densityMap) {
System.out.println("WARNING: Specifying --legacy-split will cause the first stage of the split to take much more memory! This option is considered deprecated and will be removed in a future build.");
}
maxThreads = params.getMaxThreads().getCount();
filenames = parser.getAdditionalParams();
String splitFile = params.getSplitFile();
if (splitFile != null) {
try {
areaList = new AreaList();
areaList.read(splitFile);
areaList.dump();
} catch (IOException e) {
areaList = null;
System.err.println("Could not read area list file");
e.printStackTrace();
}
}
}
/**
* Calculate the areas that we are going to split into by getting the total area and
* then subdividing down until each area has at most max-nodes nodes in it.
*/
private AreaList calculateAreas() throws IOException, XmlPullParserException {
MapCollector nodes = densityMap ? new DensityMapCollector(trim, resolution) : new NodeCollector();
MapProcessor processor = nodes;
processMap(processor);
//MapReader mapReader = processMap(processor);
//System.out.print("A total of " + Utils.format(mapReader.getNodeCount()) + " nodes, " +
// Utils.format(mapReader.getWayCount()) + " ways and " +
// Utils.format(mapReader.getRelationCount()) + " relations were processed ");
System.out.println("in " + filenames.size() + (filenames.size() == 1 ? " file" : " files"));
//System.out.println("Min node ID = " + mapReader.getMinNodeId());
//System.out.println("Max node ID = " + mapReader.getMaxNodeId());
System.out.println("Time: " + new Date());
Area exactArea = nodes.getExactArea();
SplittableArea splittableArea = nodes.getRoundedArea(resolution);
System.out.println("Exact map coverage is " + exactArea);
System.out.println("Trimmed and rounded map coverage is " + splittableArea.getBounds());
System.out.println("Splitting nodes into areas containing a maximum of " + Utils.format(maxNodes) + " nodes each...");
List<Area> areas = splittableArea.split(maxNodes);
return new AreaList(areas);
}
private void nameAreas() throws IOException {
CityFinder cityFinder;
if (geoNamesFile != null) {
CityLoader cityLoader = new CityLoader(true);
List<City> cities = cityLoader.load(geoNamesFile);
cityFinder = new DefaultCityFinder(cities);
} else {
cityFinder = new DummyCityFinder();
}
for (Area area : areaList.getAreas()) {
// Decide what to call the area
Set<City> found = cityFinder.findCities(area);
City bestMatch = null;
for (City city : found) {
if (bestMatch == null || city.getPopulation() > bestMatch.getPopulation()) {
bestMatch = city;
}
}
if (bestMatch != null)
area.setName(bestMatch.getCountryCode() + '-' + bestMatch.getName());
else
area.setName(description);
}
}
/**
* Second pass, we have the areas so parse the file(s) again and write out each element
* to the file(s) that should contain it.
*
* @param areaList Area list determined on the first pass.
*/
private void writeAreas(List<Area> areas) throws IOException, XmlPullParserException {
System.out.println("Writing out split osm files " + new Date());
int numPasses = getAreasPerPass(areas.size());
int areasPerPass = (int) Math.ceil((double) areas.size() / (double) numPasses);
if (numPasses > 1) {
System.out.println("Processing " + areas.size() + " areas in " + numPasses + " passes, " + areasPerPass + " areas at a time");
} else {
System.out.println("Processing " + areas.size() + " areas in a single pass");
}
for (int i = 0; i < numPasses; i++) {
OSMWriter[] currentWriters = new OSMWriter[Math.min(areasPerPass, areas.size() - i * areasPerPass)];
for (int j = 0; j < currentWriters.length; j++) {
Area area = areas.get(i * areasPerPass + j);
currentWriters[j] = pbfOutput ? new BinaryMapWriter(area, fileOutputDir) : new OSMXMLWriter(area, fileOutputDir);
currentWriters[j].initForWrite(area.getMapId(), overlapAmount);
}
System.out.println("Starting pass " + (i + 1) + " of " + numPasses + ", processing " + currentWriters.length +
" areas (" + areas.get(i * areasPerPass).getMapId() + " to " +
areas.get(i * areasPerPass + currentWriters.length - 1).getMapId() + ')');
MapProcessor processor = new SplitProcessor(currentWriters, maxThreads);
processMap(processor);
//System.out.println("Wrote " + Utils.format(mapReader.getNodeCount()) + " nodes, " +
// Utils.format(mapReader.getWayCount()) + " ways, " +
// Utils.format(mapReader.getRelationCount()) + " relations");
}
}
private void processMap(MapProcessor processor) throws XmlPullParserException {
// Create both an XML reader and a binary reader, Dispatch each input to the
// Appropriate parser.
OSMParser parser = new OSMParser(processor, mixed);
if (useStdIn) {
System.out.println("Reading osm data from stdin...");
Reader reader = new InputStreamReader(System.in, Charset.forName("UTF-8"));
parser.setReader(reader);
try {
try {
parser.parse();
} finally {
reader.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
for (String filename : filenames) {
System.out.println("Processing " + filename);
try {
if (filename.endsWith(".osm.pbf")) {
// Is it a binary file?
File file = new File(filename);
BlockInputStream blockinput = (new BlockInputStream(
new FileInputStream(file), new BinaryMapParser(processor)));
try {
blockinput.process();
} finally {
blockinput.close();
}
} else {
// No, try XML.
Reader reader = Utils.openFile(filename, maxThreads > 1);
parser.setReader(reader);
try {
parser.parse();
} finally {
reader.close();
}
}
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (XmlPullParserException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}
processor.endMap();
}
/**
* Write a file that can be given to mkgmap that contains the correct arguments
* for the split file pieces. You are encouraged to edit the file and so it
* contains a template of all the arguments that you might want to use.
*/
protected void writeArgsFile(List<Area> areas) {
PrintWriter w;
try {
w = new PrintWriter(new FileWriter(new File(fileOutputDir, "template.args")));
} catch (IOException e) {
System.err.println("Could not write template.args file");
return;
}
w.println("#");
w.println("# This file can be given to mkgmap using the -c option");
w.println("# Please edit it first to add a description of each map.");
w.println("#");
w.println();
w.println("# You can set the family id for the map");
w.println("# family-id: 980");
w.println("# product-id: 1");
w.println();
w.println("# Following is a list of map tiles. Add a suitable description");
w.println("# for each one.");
for (Area a : areas) {
w.println();
w.format("mapname: %08d\n", a.getMapId());
if (a.getName() == null)
w.println("# description: OSM Map");
else
w.println("description: " + a.getName());
if(pbfOutput)
w.format("input-file: %08d.osm.pbf\n", a.getMapId());
else
w.format("input-file: %08d.osm.gz\n", a.getMapId());
}
w.println();
w.close();
}
}