/*
* Copyright (C) 2006, 2011.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 3 or
* version 2 as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*/
package uk.me.parabola.mkgmap.reader.osm.boundary;
import it.unimi.dsi.fastutil.ints.IntArrayList;
import java.awt.Shape;
import java.awt.geom.PathIterator;
import java.io.BufferedOutputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.regex.Pattern;
import uk.me.parabola.log.Logger;
import uk.me.parabola.mkgmap.Version;
import uk.me.parabola.mkgmap.reader.osm.Tags;
import uk.me.parabola.util.Java2DConverter;
public class BoundarySaver
{
private static final Logger log =
Logger.
getLogger(BoundarySaver.
class);
public static final String LEGACY_DATA_FORMAT =
""; // legacy code just wrote the svn release or "svn"
public static final String RAW_DATA_FORMAT =
"RAW";
public static final String QUADTREE_DATA_FORMAT =
"QUADTREE";
public static final int CURRENT_RECORD_ID =
1;
public static final double RESET_DELTA =
Double.
POSITIVE_INFINITY;
private final File boundaryDir
;
private final String dataFormat
;
private uk.
me.
parabola.
imgfmt.
app.
Area bbox
;
private final HashSet<String> writtenFileNames
;
private int minLat =
Integer.
MAX_VALUE;
private int minLong =
Integer.
MAX_VALUE;
private int maxLat =
Integer.
MIN_VALUE;
private int maxLong =
Integer.
MIN_VALUE;
private static final class StreamInfo
{
File file
;
String boundsKey
;
OutputStream stream
;
int lastAccessNo
;
public StreamInfo
() {
this.
lastAccessNo =
0;
}
public boolean isOpen
() {
return stream
!=
null;
}
public void close
() {
if (stream
!=
null) {
try {
stream.
close();
} catch (IOException exp
) {
log.
error(exp
);
}
}
stream =
null;
}
}
private int lastAccessNo =
0;
private final List<StreamInfo
> openStreams =
new ArrayList<>();
/** keeps the open streams */
private final Map<String, StreamInfo
> streams
;
private boolean createEmptyFiles =
false;
public BoundarySaver
(File boundaryDir,
String mode
) {
this.
boundaryDir = boundaryDir
;
if (boundaryDir.
exists() && boundaryDir.
isDirectory() ==
false){
log.
error("output target exists and is not a directory");
System.
exit(-
1);
}
this.
dataFormat = mode
;
this.
streams =
new HashMap<>();
this.
writtenFileNames =
new HashSet<>();
}
/**
* Saves the given BoundaryQuadTree to a stream
* @param bqt the BoundaryQuadTree
* @param boundsFileName the file name
*/
public void saveQuadTree
(BoundaryQuadTree bqt,
String boundsFileName
) {
String[] parts = boundsFileName.
split("[_" +
Pattern.
quote(".") +
"]");
String key = boundsFileName
;
if (parts.
length >=
3) {
key = parts
[1] +
"_" + parts
[2];
}
try {
StreamInfo streamInfo = getStream
(key
);
if (streamInfo
!=
null && streamInfo.
isOpen()) {
bqt.
save(streamInfo.
stream);
writtenFileNames.
add(boundsFileName
);
}
} catch (Exception exp
) {
log.
error("Cannot write boundary: " + exp, exp
);
}
tidyStreams
();
}
public void addBoundary
(Boundary boundary
) {
Map<String,
Shape> splitBounds = BoundaryUtil.
rasterArea(boundary.
getArea());
for (Entry
<String,
Shape> split : splitBounds.
entrySet()) {
saveToFile
(split.
getKey(), split.
getValue(), boundary.
getTags(),
boundary.
getId());
}
}
public HashSet<String> end
() {
if (isCreateEmptyFiles
() && getBbox
() !=
null) {
// a bounding box is set => fill the gaps with empty files
for (int latSplit = BoundaryUtil.
getSplitBegin(getBbox
()
.
getMinLat()); latSplit
<= BoundaryUtil
.
getSplitBegin(getBbox
().
getMaxLat()); latSplit += BoundaryUtil.
RASTER) {
for (int lonSplit = BoundaryUtil.
getSplitBegin(getBbox
()
.
getMinLong()); lonSplit
<= BoundaryUtil
.
getSplitBegin(getBbox
().
getMaxLong()); lonSplit += BoundaryUtil.
RASTER) {
String key = BoundaryUtil.
getKey(latSplit, lonSplit
);
// check if the stream already exist but do no open it
StreamInfo stream = getStream
(key,
false);
if (stream ==
null) {
// it does not exist => create a new one to write out
// the common header of the boundary file
stream = getStream
(key
);
}
// close the stream if it is open
if (stream.
isOpen())
stream.
close();
streams.
remove(key
);
}
}
}
// close the rest of the streams
for (StreamInfo streamInfo : streams.
values()) {
streamInfo.
close();
}
streams.
clear();
openStreams.
clear();
return writtenFileNames
;
}
private void tidyStreams
() {
if (openStreams.
size() < 100) {
return;
}
Collections.
sort(openStreams,
new Comparator<StreamInfo
>() {
public int compare
(StreamInfo o1, StreamInfo o2
) {
return o1.
lastAccessNo - o2.
lastAccessNo;
}
});
log.
debug(openStreams.
size(),
"open streams.");
List<StreamInfo
> closingStreams = openStreams.
subList(0,
openStreams.
size() -
80);
// close and remove the streams from the open list
for (StreamInfo streamInfo : closingStreams
) {
log.
debug("Closing", streamInfo.
file);
streamInfo.
close();
}
closingStreams.
clear();
log.
debug("Remaining", openStreams.
size(),
"open streams.");
}
private void openStream
(StreamInfo streamInfo,
boolean newFile
) {
if (streamInfo.
file.
getParentFile().
exists() ==
false
&& streamInfo.
file.
getParentFile() !=
null)
streamInfo.
file.
getParentFile().
mkdirs();
FileOutputStream fileStream =
null;
try {
fileStream =
new FileOutputStream(streamInfo.
file,
!newFile
);
streamInfo.
stream =
new BufferedOutputStream(fileStream
);
openStreams.
add(streamInfo
);
if (newFile
) {
writeDefaultInfos
(streamInfo.
stream);
String[] keyParts = streamInfo.
boundsKey.
split(Pattern
.
quote("_"));
int lat =
Integer.
parseInt(keyParts
[0]);
int lon =
Integer.
parseInt(keyParts
[1]);
if (lat
< minLat
) {
minLat = lat
;
log.
debug("New min Lat:", minLat
);
}
if (lat
> maxLat
) {
maxLat = lat
;
log.
debug("New max Lat:", maxLat
);
}
if (lon
< minLong
) {
minLong = lon
;
log.
debug("New min Lon:", minLong
);
}
if (lon
> maxLong
) {
maxLong = lon
;
log.
debug("New max Long:", maxLong
);
}
}
} catch (IOException exp
) {
log.
error("Cannot save boundary: " + exp
);
if (fileStream
!=
null) {
try {
fileStream.
close();
} catch (Throwable thr
) {
}
}
}
}
private StreamInfo getStream
(String filekey
) {
return getStream
(filekey,
true);
}
private StreamInfo getStream
(String filekey,
boolean autoopen
) {
StreamInfo stream = streams.
get(filekey
);
if (autoopen
) {
if (stream ==
null) {
log.
debug("Create stream for", filekey
);
stream =
new StreamInfo
();
stream.
boundsKey = filekey
;
stream.
file =
new File(boundaryDir,
"bounds_" + filekey
+
".bnd");
streams.
put(filekey, stream
);
openStream
(stream,
true);
} else if (stream.
isOpen() ==
false) {
openStream
(stream,
false);
}
}
if (stream
!=
null) {
stream.
lastAccessNo = ++lastAccessNo
;
}
return stream
;
}
private void writeDefaultInfos
(OutputStream stream
) throws IOException {
DataOutputStream dos =
new DataOutputStream(stream
);
dos.
writeUTF("BND");
dos.
writeLong(System.
currentTimeMillis());
// write the header part 2
// write it first to a byte array to be able to calculate the length of the header
ByteArrayOutputStream headerStream =
new ByteArrayOutputStream();
try(DataOutputStream headerDataStream =
new DataOutputStream(headerStream
)){
headerDataStream.
writeUTF(dataFormat
);
headerDataStream.
writeInt(CURRENT_RECORD_ID
);
headerDataStream.
writeUTF(Version.
VERSION);
}
byte[] header2 = headerStream.
toByteArray();
// write the length of the header part 2 so that it is possible to add
// values in the future
dos.
writeInt(header2.
length);
dos.
write(header2
);
dos.
flush();
}
/**
* Save the elements that build a boundary with a given key
* that identifies the lower left corner of the raster.
* @param filekey the string that identifies the lower left corner
* @param shape the shape that describes the area of the boundary
* @param tags the tags of the boundary
* @param id the boundary id
*/
private void saveToFile
(String filekey,
Shape shape, Tags tags,
String id
) {
try {
StreamInfo streamInfo = getStream
(filekey
);
if (streamInfo
!=
null && streamInfo.
isOpen()) {
writeRawFormat
(streamInfo.
stream, shape, tags, id
);
}
} catch (Exception exp
) {
log.
error("Cannot write boundary: " + exp, exp
);
}
tidyStreams
();
}
/**
* Save the elements of a boundary to a stream.
* @param stream the already opened OutputStream
* @param shape the shape that describes the area of the boundary
* @param tags the tags of the boundary
* @param id the boundary id
*/
private void writeRawFormat
(OutputStream stream,
Shape shape, Tags tags,
String id
) {
ByteArrayOutputStream oneItemStream =
new ByteArrayOutputStream();
DataOutputStream dos =
new DataOutputStream(oneItemStream
);
if (dataFormat == QUADTREE_DATA_FORMAT
) {
log.
error("wrong format for write, must use BoundaryQuadTree.save() ");
System.
exit(1);
}
try {
dos.
writeUTF(id
);
// write the tags
int noOfTags = tags.
size();
dos.
writeInt(noOfTags
);
Iterator<Entry
<String,
String>> tagIter = tags.
entryIterator();
while (tagIter.
hasNext()) {
Entry
<String,
String> tag = tagIter.
next();
dos.
writeUTF(tag.
getKey());
dos.
writeUTF(tag.
getValue());
noOfTags--
;
}
assert noOfTags ==
0 :
"Remaining tags: " + noOfTags +
" size: "
+ tags.
size() +
" "
+ tags.
toString();
//writeArea(dos,boundary.getArea());
writeArea
(dos, shape
);
dos.
close();
// now start to write into the real stream
// first write the bounding box so that is possible to skip the
// complete entry
uk.
me.
parabola.
imgfmt.
app.
Area outBBox = Java2DConverter
.
createBbox(shape
);
DataOutputStream dOutStream =
new DataOutputStream(stream
);
dOutStream.
writeInt(outBBox.
getMinLat());
dOutStream.
writeInt(outBBox.
getMinLong());
dOutStream.
writeInt(outBBox.
getMaxLat());
dOutStream.
writeInt(outBBox.
getMaxLong());
// write the size of the boundary block so that it is possible to
// skip it
byte[] data = oneItemStream.
toByteArray();
assert data.
length > 0 :
"bSize is not > 0 : " + data.
length;
dOutStream.
writeInt(data.
length);
// write the boundary block
dOutStream.
write(data
);
dOutStream.
flush();
} catch (IOException exp
) {
log.
error(exp.
toString());
}
}
/**
* Write area to stream with Double precision. The coordinates
* are saved as varying length doubles with delta coding.
* @param dos the already opened DataOutputStream
* @param area the area (can be non-singular)
* @throws IOException
*/
public static void writeArea
(DataOutputStream dos,
Shape area
) throws IOException{
double[] res =
new double[6];
double[] lastRes =
new double[2];
IntArrayList pairs =
new IntArrayList
();
// step 1: count parts
PathIterator pit = area.
getPathIterator(null);
int prevType = -
1;
int len =
0;
while (!pit.
isDone()) {
int type = pit.
currentSegment(res
);
if (type
!=
PathIterator.
SEG_LINETO && prevType ==
PathIterator.
SEG_LINETO){
pairs.
add(len
);
len =
0;
}
if (type ==
PathIterator.
SEG_LINETO)
len++
;
prevType = type
;
pit.
next();
}
// 2nd pass: write the data
pit = area.
getPathIterator(null);
prevType = -
1;
int pairsPos =
0;
dos.
writeInt(pit.
getWindingRule());
while (!pit.
isDone()) {
int type = pit.
currentSegment(res
);
if (type
!= prevType
)
dos.
writeInt(type
);
switch (type
) {
case PathIterator.
SEG_LINETO:
if (prevType
!= type
){
len = pairs.
getInt(pairsPos++
);
dos.
writeInt(len
);
}
// no break
//$FALL-THROUGH$
case PathIterator.
SEG_MOVETO:
len--
;
for (int i =
0; i
< 2; i++
){
double delta = res
[i
] - lastRes
[i
];
if (delta + lastRes
[i
] != res
[i
]){
// handle rounding error in delta processing
// write POSITIVE_INFINITY to signal that next value is
// not delta coded
//System.out.println("reset " + i ) ;
writeVarDouble
(dos, BoundarySaver.
RESET_DELTA);
delta = res
[i
];
}
lastRes
[i
] = res
[i
];
writeVarDouble
(dos, delta
);
}
break;
case PathIterator.
SEG_CLOSE:
break;
default:
log.
error("Unsupported path iterator type " + type
+
". This is an mkgmap error.");
}
prevType = type
;
pit.
next();
}
if (len
!=
0){
log.
error("len not zero " + len
);
}
dos.
writeInt(-
1); // isDone flag
}
public uk.
me.
parabola.
imgfmt.
app.
Area getBbox
() {
if (bbox ==
null) {
bbox =
new uk.
me.
parabola.
imgfmt.
app.
Area(minLat, minLong, maxLat,
maxLong
);
log.
error("Calculate bbox to " + bbox
);
}
return bbox
;
}
public void setBbox
(uk.
me.
parabola.
imgfmt.
app.
Area bbox
) {
if (bbox.
isEmpty()) {
log.
warn("Do not use bounding box because it's empty");
this.
bbox =
null;
} else {
this.
bbox = bbox
;
log.
info("Set bbox: " + bbox.
getMinLat() +
" " + bbox.
getMinLong()
+
" " + bbox.
getMaxLat() +
" " + bbox.
getMaxLong());
}
}
public boolean isCreateEmptyFiles
() {
return createEmptyFiles
;
}
/**
* Sets if empty bounds files should be created for areas without any
* boundary. Typically these are sea areas or areas not included in the OSM
* file.
*
* @param createEmptyFiles
* <code>true</code> create bounds files for uncovered areas;
* <code>false</code> create bounds files only for areas
* containing boundary information
*/
public void setCreateEmptyFiles
(boolean createEmptyFiles
) {
this.
createEmptyFiles = createEmptyFiles
;
}
/**
* Write a varying length double. A typical double value requires only ~ 20 bits
* (the left ones). As in o5m format we use the leftmost bit of a byte to signal
* that a further byte is to read, the remaining 7 bits are used to store the value.
* Many values are stored within 3 bytes, but some may require 10 bytes
* (64 bits = 9*7 + 1) . We start with the highest bits of the long value that
* represents the double.
*
* @param dos the already opened OutputStream
* @param val the double value to be written
* @throws IOException
*/
private static void writeVarDouble
(OutputStream dos,
double val
) throws IOException{
long v64 =
Double.
doubleToRawLongBits(val
);
if (v64 ==
0){
dos.
write(0);
return;
}
byte[] buffer =
new byte[12];
int numBytes =
0;
while(v64
!=
0){
v64 =
(v64
<< 7) |
(v64
>>> -
7); // rotate left 7 bits
buffer
[numBytes++
] =
(byte)(v64
& 0x7f|0x80L
);
v64
&= 0xffffffffffffff80L
;
}
buffer
[numBytes-
1] &= 0x7f
;
dos.
write(buffer,
0, numBytes
);
}
}