2 * Navit, a modular navigation system.
3 * Copyright (C) 2005-2008 Navit Team
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License
7 * version 2 as published by the Free Software Foundation.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the
16 * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
17 * Boston, MA 02110-1301, USA.
21 #define _FILE_OFFSET_BITS 64
22 #define _LARGEFILE_SOURCE
23 #define _LARGEFILE64_SOURCE
39 #ifdef HAVE_POSTGRESQL
43 #define BUFFER_SIZE 1280
45 #define GENERATE_INDEX
48 #define debug_tile(x) 0
50 #define debug_tile(x) (!strcmp(x,"bcdbd") || !strcmp(x,"bcdbd") || !strcmp(x,"bcdbda") || !strcmp(x,"bcdbdb") || !strcmp(x,"bcdbdba") || !strcmp(x,"bcdbdbb") || !strcmp(x,"bcdbdbba") || !strcmp(x,"bcdbdbaa") || !strcmp(x,"bcdbdbacaa") || !strcmp(x,"bcdbdbacab") || !strcmp(x,"bcdbdbacaba") || !strcmp(x,"bcdbdbacabaa") || !strcmp(x,"bcdbdbacabab") || !strcmp(x,"bcdbdbacababb") || !strcmp(x,"bcdbdbacababba") || !strcmp(x,"bcdbdbacababbb") || !strcmp(x,"bcdbdbacababbd") || !strcmp(x,"bcdbdbacababaa") || !strcmp(x,"bcdbdbacababab") || !strcmp(x,"bcdbdbacababac") || !strcmp(x,"bcdbdbacababad") || !strcmp(x,"bcdbdbacabaaa") || !strcmp(x,"bcdbdbacabaaba") || !strcmp(x,"bcdbdbacabaabb") || !strcmp(x,"bcdbdbacabaabc") || !strcmp(x,"bcdbdbacabaabd") || !strcmp(x,"bcdbdbacabaaaa") || !strcmp(x,"bcdbdbacabaaab") || !strcmp(x,"bcdbdbacabaaac") || !strcmp(x,"bcdbdbacabaaad") || 0)
54 static GHashTable *dedupe_ways_hash;
56 static int attr_debug_level=1;
57 static int nodeid,wayid;
58 static int report,phase;
59 static int ignore_unkown = 0, coverage=0;
61 static char *attrmap={
63 "n amenity=hospital poi_hospital\n"
64 "n amenity=atm poi_bank\n"
65 "n amenity=bank poi_bank\n"
66 "n amenity=pub poi_bar\n"
67 "n amenity=cafe poi_cafe\n"
68 "n amenity=bus_station poi_bus_station\n"
69 "n amenity=parking poi_car_parking\n"
70 "n amenity=cinema poi_cinema\n"
71 "n amenity=fire_station poi_firebrigade\n"
72 "n amenity=fuel poi_fuel\n"
73 "n amenity=courthouse poi_justice\n"
74 "n amenity=library poi_library\n"
75 "n amenity=pharmacy poi_pharmacy\n"
76 "n amenity=place_of_worship,religion=christian poi_church\n"
77 "n amenity=police poi_police\n"
78 "n amenity=post_office poi_post\n"
79 "n amenity=post_box poi_post\n"
80 "n amenity=public_building poi_public_office\n"
81 "n amenity=restaurant poi_restaurant\n"
82 "n amenity=fast_food poi_fastfood\n"
83 "n amenity=toilets poi_restroom\n"
84 "n amenity=school poi_school\n"
85 "n amenity=university poi_school\n"
86 "n amenity=college poi_school\n"
87 "n amenity=telephone poi_telephone\n"
88 "n amenity=theatre poi_theater\n"
89 "n amenity=townhall poi_townhall\n"
90 "n highway=bus_stop poi_bus_stop\n"
91 "n highway=mini_roundabout mini_roundabout\n"
92 "n highway=motorway_junction highway_exit\n"
93 "n highway=traffic_signals traffic_signals\n"
94 "n highway=turning_circle turning_circle\n"
95 "n leisure=slipway poi_boat_ramp\n"
96 "n leisure=fishing poi_fish\n"
97 "n sport=golf poi_golf\n"
98 "n leisure=golf_course poi_golf\n"
99 "n leisure=marina poi_marine\n"
100 "n leisure=sports_centre poi_sport\n"
101 "n leisure=stadium poi_stadium\n"
102 "n shop=supermarket poi_shopping\n"
103 "n shop=convenience poi_shop_grocery\n"
104 "n tourism=attraction poi_attraction\n"
105 "n tourism=camp_site poi_camp_rv\n"
106 "n tourism=caravan_site poi_camp_rv\n"
107 "n tourism=hotel poi_hotel\n"
108 "n tourism=motel poi_hotel\n"
109 "n tourism=guest_house poi_hotel\n"
110 "n tourism=hostel poi_hotel\n"
111 "n tourism=information poi_information\n"
112 "n tourism=museum poi_museum_history\n"
113 "n tourism=picnic_site poi_picnic\n"
114 "n tourism=theme_park poi_resort\n"
115 "n tourism=zoo poi_zoo\n"
116 "n amenity=grave_yard poi_cemetery\n"
117 "n landuse=cemetery poi_cemetery\n"
118 "n military=airfield poi_military\n"
119 "n military=bunker poi_military\n"
120 "n military=barracks poi_military\n"
121 "n military=range poi_military\n"
122 "n military=danger_area poi_danger_area\n"
123 "n sport=swimming poi_swimming\n"
124 "n sport=skiing poi_skiing\n"
125 "n aeroway=aerodrome poi_airport\n"
126 "n aeroway=airport poi_airport\n"
127 "n aeroway=terminal poi_airport\n"
128 "n aeroway=helipad poi_heliport\n"
129 "n man_made=tower poi_tower\n"
130 "n natural=bay poi_bay\n"
131 "n natural=peak poi_peak\n"
132 "n place=suburb district_label\n"
133 "n place=city town_label_2e5\n"
134 "n place=town town_label_2e4\n"
135 "n place=village town_label_2e3\n"
136 "n place=hamlet town_label_2e2\n"
137 "n place=locality town_label_2e0\n"
138 "n railway=halt poi_rail_halt\n"
139 "n railway=level_crossing poi_level_crossing\n"
140 "n railway=station poi_rail_station\n"
141 "n railway=tram_stop poi_rail_tram_stop\n"
143 "n poi_bench amenity=park_bench amenity=bench\n"
144 "n poi_biergarten amenity=biergarten\n"
145 "n poi_boundary_stone historic=boundary_stone\n"
146 "n poi_castle historic=castle\n"
147 "n poi_hunting_stand Annehmlichkeit=Hochsitz\n"
148 "n poi_memorial historic=memorial\n"
149 "n poi_monument historic=monument\n"
150 "n poi_shelter amenity=shelter\n"
151 "n poi_fountain amenity=fountain\n"
152 "n poi_potable_water amenity=drinking_water\n"
153 "n poi_toilets amenity=toilets\n"
154 "n poi_viewpoint tourism=viewpoint\n"
155 "n amenity=park_bench poi_bench\n"
156 "n amenity=bench poi_bench\n"
157 "n amenity=biergarten poi_biergarten\n"
158 "n historic=boundary_stone poi_boundary_stone\n"
159 "n historic=castle poi_castle\n"
160 "n Annehmlichkeit=Hochsitz poi_hunting_stand\n"
161 "n historic=memorial poi_memorial\n"
162 "n historic=monument poi_monument\n"
163 "n amenity=shelter poi_shelter\n"
164 "n amenity=fountain poi_fountain\n"
165 "n amenity=drinking_water poi_potable_water\n"
166 "n amenity=toilets poi_toilets\n"
167 "n tourism=viewpoint poi_viewpoint\n"
168 "w *=* street_unkn\n"
169 "w amenity=place_of_worship poly_building\n"
170 "w building=* poly_building\n"
171 "w aeroway=aerodrome poly_airport\n"
172 "w aeroway=apron poly_apron\n"
173 "w aeroway=runway aeroway_runway\n"
174 "w aeroway=taxiway aeroway_taxiway\n"
175 "w aeroway=terminal poly_terminal\n"
176 "w highway=bridleway bridleway\n"
177 "w highway=path,horse=designated bridleway\n"
178 "w highway=cycleway cycleway\n"
179 "w highway=cyclepath cycleway\n"
180 "w highway=path,bicycle=designated cycleway\n"
181 "w highway=footway footway\n"
182 "w piste:type=nordic piste_nordic\n"
183 "w route=ski piste_nordic\n"
184 "w piste:type=downhill,piste:difficulty=novice piste_downhill_novice\n"
185 "w piste:type=downhill,piste:difficulty=easy piste_downhill_easy\n"
186 "w piste:type=downhill,piste:difficulty=intermediate piste_downhill_intermediate\n"
187 "w piste:type=downhill,piste:difficulty=advanced piste_downhill_advanced\n"
188 "w piste:type=downhill,piste:difficulty=expert piste_downhill_expert\n"
189 "w piste:type=downhill,piste:difficulty=freeride piste_downhill_freeride\n"
190 "w highway=path,foot=designated footway\n"
191 "w highway=steps steps\n"
192 "w highway=path path\n"
193 "w highway=path,sac_scale=hiking hiking\n"
194 "w highway=path,sac_scale=mountain_hiking hiking_mountain\n"
195 "w highway=path,sac_scale=demanding_mountain_hiking hiking_mountain_demanding\n"
196 "w highway=path,sac_scale=alpine_hiking hiking_alpine\n"
197 "w highway=path,sac_scale=demanding_alpine_hiking hiking_alpine_demanding\n"
198 "w highway=path,sac_scale=difficult_alpine_hiking hiking_alpine_difficult\n"
199 "w highway=unsurfaced track_gravelled\n"
200 "w highway=track track_gravelled\n"
201 "w highway=track,tracktype=grade1 track_paved\n"
202 "w highway=track,surface=paved track_paved\n"
203 "w highway=track,tracktype=grade2 track_gravelled\n"
204 "w highway=track,surface=gravel track_gravelled\n"
205 "w highway=track,tracktype=grade3 track_unpaved\n"
206 "w highway=track,surface=unpaved track_unpaved\n"
207 "w highway=track,tracktype=grade4 track_ground\n"
208 "w highway=track,surface=ground track_ground\n"
209 "w highway=track,tracktype=grade5 track_grass\n"
210 "w highway=track,surface=grass track_grass\n"
211 "w highway=parking_lane street_parking_lane\n"
212 "w highway=service street_service\n"
213 "w highway=service,area=1 poly_service\n"
214 "w highway=construction street_construction\n"
215 "w highway=pedestrian street_pedestrian\n"
216 "w highway=pedestrian,area=1 poly_pedestrian\n"
217 "w highway=plaza poly_plaza\n"
218 "w landuse=plaza poly_plaza\n"
219 "w highway=residential street_1_city\n"
220 "w highway=residential,area=1 poly_street_1\n"
221 "w highway=living_street living_street\n"
222 "w highway=unclassified street_1_city\n"
223 "w highway=unclassified,area=1 poly_street_1\n"
224 "w highway=road street_1_city\n"
225 "w highway=minor street_1_land\n"
226 "w highway=tertiary street_2_city\n"
227 "w highway=tertiary,area=1 poly_street_2\n"
228 "w highway=tertiary_link ramp\n"
229 "w highway=secondary street_3_city\n"
230 "w highway=secondary,area=1 poly_street_3\n"
231 "w highway=secondary_link ramp\n"
232 "w highway=primary street_4_city\n"
233 "w highway=primary_link ramp\n"
234 "w highway=trunk street_4_city\n"
235 "w highway=trunk_link ramp\n"
236 "w highway=motorway highway_city\n"
237 "w highway=motorway_link ramp\n"
238 "w historic=town gate poly_building\n"
239 "w cycleway=track cycleway\n"
240 "w amenity=parking poly_car_parking\n"
241 "w landuse=allotments poly_allotments\n"
242 "w landuse=cemetery poly_cemetery\n"
243 "w amenity=grave_yard poly_cemetery\n"
244 "w landuse=forest poly_wood\n"
245 "w landuse=industrial poly_industry\n"
246 "w landuse=residential poly_town\n"
247 "w landuse=residential,area=1 poly_town\n"
248 "w landuse=farmyard poly_town\n"
249 "w amenity=college poly_college\n"
250 "w amenity=university poly_university\n"
251 "w tourism=camp_site poly_camp_site\n"
252 "w tourism=caravan_site poly_caravan_site\n"
253 "w tourism=picnic_site poly_picnic_site\n"
254 "w tourism=theme_park poly_theme_park\n"
255 "w tourism=attraction poly_attraction\n"
256 "w tourism=zoo poly_zoo\n"
257 "w tourism=artwork poly_artwork\n"
258 "w historic=archaeological_site poly_archaeological_site\n"
259 "w historic=ruins poly_ruins\n"
260 "w historic=battlefield poly_battlefield\n"
261 "w landuse=quarry poly_quarry\n"
262 "w landuse=landfill poly_landfill\n"
263 "w landuse=retail poly_retail\n"
264 "w landuse=commercial poly_commercial\n"
265 "w landuse=brownfield poly_brownfield\n"
266 "w landuse=greenfield poly_greenfield\n"
267 "w landuse=construction poly_construction\n"
268 "w landuse=railway poly_railway\n"
269 "w natural=glacier poly_glacier\n"
270 "w natural=scree poly_scree\n"
271 "w natural=scrub poly_scrub\n"
272 "w natural=fell poly_fell\n"
273 "w natural=heath poly_heath\n"
274 "w natural=marsh poly_marsh\n"
275 "w natural=mud poly_mud\n"
276 "w natural=beach poly_beach\n"
277 "w natural=land poly_land\n"
278 "w landuse=basin poly_basin\n"
279 "w landuse=reservoir poly_reservoir\n"
280 "w landuse=farm poly_farm\n"
281 "w landuse=farmland poly_farm\n"
282 "w landuse=vineyard poly_farm\n"
283 "w leisure=park poly_park\n"
284 "w landuse=village_green poly_village_green\n"
285 "w landuse=recreation_ground poly_recreation_ground\n"
286 "w natural=wood poly_wood\n"
287 "w natural=water poly_water\n"
288 "w natural=coastline water_line\n"
289 "w place=suburb poly_town\n"
290 "w place=town poly_town\n"
291 "w landuse=military poly_military\n"
292 "w military=airfield poly_airfield\n"
293 "w military=barracks poly_barracks\n"
294 "w military=danger_area poly_danger_area\n"
295 "w military=range poly_range\n"
296 "w military=naval_base poly_naval_base\n"
297 "w power=line powerline\n"
298 "w railway=rail rail\n"
299 "w railway=narrow_gauge rail_narrow_gauge\n"
300 "w railway=light_rail rail_light\n"
301 "w railway=subway rail_subway\n"
302 "w railway=tram rail_tram\n"
303 "w railway=monorail rail_mono\n"
304 "w railway=preserved rail_preserved\n"
305 "w railway=disused rail_disused\n"
306 "w railway=abandoned rail_abandoned\n"
307 "w aerialway=cable_car lift_cable_car\n"
308 "w aerialway=chair_lift lift_chair\n"
309 "w aerialway=drag_lift lift_drag\n"
310 "w leisure=golf_course poly_golf_course\n"
311 "w sport=* poly_sport\n"
312 "w leisure=sports_centre poly_sport\n"
313 "w leisure=stadium poly_sports_stadium\n"
314 "w leisure=track poly_sports_track\n"
315 "w leisure=pitch poly_sports_pitch\n"
316 "w leisure=water_park poly_water_park\n"
317 "w leisure=marina poly_marina\n"
318 "w leisure=fishing poly_fishing\n"
319 "w leisure=garden poly_garden\n"
320 "w leisure=common poly_common\n"
321 "w leisure=playground poly_playground\n"
322 "w leisure=nature_reserve poly_nature_reserve\n"
323 "w waterway=canal water_canal\n"
324 "w waterway=river water_river\n"
325 "w waterway=stream water_stream\n"
326 "w waterway=drain water_drain\n"
327 "w waterway=riverbank poly_water\n"
328 "w boundary=administrative border_country\n"
329 "w boundary=civil border_civil\n"
330 "w boundary=political border_political\n"
331 "w boundary=national_park border_national_park\n"
332 "w route=ferry ferry\n"
333 "w highway=bus_guideway bus_guideway\n"
339 } coord_buffer[65536];
341 #define IS_REF(c) ((c).x >= (1 << 30))
342 #define REF(c) ((c).y)
343 #define SET_REF(c,ref) do { (c).x = 1 << 30; (c).y = ref ; } while(0)
349 static void bbox_extend(struct coord *c, struct rect *r);
351 #ifdef GENERATE_INDEX
353 static GHashTable *aux_tile_hash;
354 GList *aux_tile_list;
356 struct country_table {
363 } country_table[] = {
364 { 40,"Austria,Österreich,AUT"},
366 {276,"Germany,Deutschland,Bundesrepublik Deutschland"},
367 {528,"Nederland,The Netherlands,Niederlande,NL"},
372 static GHashTable *country_table_hash;
375 struct attr_mapping {
377 int attr_present_idx_count;
378 int attr_present_idx[0];
381 static struct attr_mapping **attr_mapping_node;
382 static int attr_mapping_node_count;
383 static struct attr_mapping **attr_mapping_way;
384 static int attr_mapping_way_count;
386 static char *attr_present;
387 static int attr_present_count;
388 static GHashTable *attr_hash;
391 static GHashTable *strings_hash = NULL;
394 static char* string_hash_lookup( const char* key )
396 char* key_ptr = NULL;
398 if ( strings_hash == NULL ) {
399 strings_hash = g_hash_table_new(g_str_hash, g_str_equal);
402 if ( ( key_ptr = g_hash_table_lookup(strings_hash, key )) == NULL ) {
403 key_ptr = g_strdup( key );
404 g_hash_table_insert(strings_hash, key_ptr, (gpointer)key_ptr );
411 build_attrmap_line(char *line)
413 char *t=NULL,*kvl=NULL,*i=NULL,*p,*kv;
414 struct attr_mapping ***attr_mapping_curr,*attr_mapping=g_malloc0(sizeof(struct attr_mapping));
415 int idx,attr_mapping_count=0,*attr_mapping_curr_count;
432 attr_mapping_curr=&attr_mapping_way;
433 attr_mapping_curr_count=&attr_mapping_way_count;
437 attr_mapping_curr=&attr_mapping_node;
438 attr_mapping_curr_count=&attr_mapping_node_count;
440 attr_mapping->type=item_from_name(i);
441 while ((kv=strtok(kvl, ","))) {
443 if (!(idx=(int)g_hash_table_lookup(attr_hash, kv))) {
444 idx=attr_present_count++;
445 g_hash_table_insert(attr_hash, kv, (gpointer) idx);
447 attr_mapping=g_realloc(attr_mapping, sizeof(struct attr_mapping)+(attr_mapping_count+1)*sizeof(int));
448 attr_mapping->attr_present_idx[attr_mapping_count++]=idx;
449 attr_mapping->attr_present_idx_count=attr_mapping_count;
451 *attr_mapping_curr=g_realloc(*attr_mapping_curr, sizeof(**attr_mapping_curr)*(*attr_mapping_curr_count+1));
452 (*attr_mapping_curr)[(*attr_mapping_curr_count)++]=attr_mapping;
456 build_attrmap(char *map)
459 attr_hash=g_hash_table_new(g_str_hash, g_str_equal);
460 attr_present_count=1;
466 build_attrmap_line(map);
469 attr_present=g_malloc0(sizeof(*attr_present)*attr_present_count);
472 #ifdef GENERATE_INDEX
474 build_countrytable(void)
477 char *names,*str,*tok;
478 country_table_hash=g_hash_table_new(g_str_hash, g_str_equal);
479 for (i = 0 ; i < sizeof(country_table)/sizeof(struct country_table) ; i++) {
480 names=g_strdup(country_table[i].names);
482 while ((tok=strtok(str, ","))) {
484 g_hash_table_insert(country_table_hash, tok, (gpointer)&country_table[i]);
492 static int processed_nodes, processed_nodes_out, processed_ways, processed_relations, processed_tiles;
493 static int in_way, in_node, in_relation;
499 signal(SIGALRM, sig_alrm);
502 fprintf(stderr,"PROGRESS%d: Processed %d nodes (%d out) %d ways %d relations %d tiles\n", phase, processed_nodes, processed_nodes_out, processed_ways, processed_relations, processed_tiles);
516 struct attr_bin label_attr = {
519 char label_attr_buffer[BUFFER_SIZE];
521 struct attr_bin town_name_attr = {
525 struct attr_bin street_name_attr = {
529 struct attr_bin street_name_systematic_attr = {
530 0, attr_street_name_systematic
532 char street_name_systematic_attr_buffer[BUFFER_SIZE];
534 struct attr_bin debug_attr = {
537 char debug_attr_buffer[BUFFER_SIZE];
539 struct attr_bin flags_attr = {
542 int flags_attr_value;
544 char is_in_buffer[BUFFER_SIZE];
547 static void write_zipmember(FILE *out, FILE *dir_out, char *name, int filelen, char *data, int data_size, int compression_level);
550 pad_text_attr(struct attr_bin *a, char *buffer)
553 if (buffer && buffer[0]) {
563 xml_get_attribute(char *xml, char *attribute, char *buffer, int buffer_size)
565 int len=strlen(attribute);
566 char *pos,*i,s,attr[len+2];
567 strcpy(attr, attribute);
568 strcpy(attr+len, "=");
569 pos=strstr(xml, attr);
579 if (i - pos > buffer_size) {
580 fprintf(stderr,"Buffer overflow %d vs %d\n", i-pos, buffer_size);
583 strncpy(buffer, pos, i-pos);
588 static int node_is_tagged;
591 add_tag(char *k, char *v)
593 GHashTable *value_hash;
596 char buffer[BUFFER_SIZE*2+2];
597 if (! strcmp(k,"ele"))
599 if (! strcmp(k,"time"))
601 if (! strcmp(k,"created_by"))
603 if (! strncmp(k,"tiger:",6) || !strcmp(k,"AND_nodes"))
605 if (! strcmp(k,"converted_by") || ! strcmp(k,"source"))
607 if (! strncmp(k,"osmarender:",11) || !strncmp(k,"svg:",4))
609 if (! strcmp(k,"layer"))
611 if (! strcasecmp(v,"true") || ! strcasecmp(v,"yes"))
613 if (! strcmp(k,"oneway")) {
614 if (!strcmp(v,"1")) {
615 flags_attr_value=AF_ONEWAY;
618 if (! strcmp(v,"-1")) {
619 flags_attr_value=AF_ONEWAYREV;
627 if (! strcmp(k,"junction")) {
628 if (! strcmp(v,"roundabout")) {
629 flags_attr_value=AF_ONEWAY;
633 if (! strcmp(k,"maxspeed")) {
636 if (! strcmp(k,"bicycle")) {
639 if (! strcmp(k,"foot")) {
642 if (! strcmp(k,"note"))
644 if (! strcmp(k,"name")) {
645 strcpy(label_attr_buffer, v);
646 pad_text_attr(&label_attr, label_attr_buffer);
649 if (! strcmp(k,"ref")) {
651 strcpy(street_name_systematic_attr_buffer, v);
652 pad_text_attr(&street_name_systematic_attr, street_name_systematic_attr_buffer);
656 if (! strcmp(k,"is_in")) {
657 strcpy(is_in_buffer, v);
660 if (! strcmp(k,"gnis:ST_alpha")) {
661 /* assume a gnis tag means it is part of the USA:
662 http://en.wikipedia.org/wiki/Geographic_Names_Information_System
663 many US towns do not have is_in tags
665 strcpy(is_in_buffer, "USA");
668 if (! strcmp(k,"lanes")) {
671 if (attr_debug_level >= level) {
672 int bytes_left = sizeof( debug_attr_buffer ) - strlen(debug_attr_buffer) - 1;
673 if ( bytes_left > 0 )
675 snprintf(debug_attr_buffer+strlen(debug_attr_buffer), bytes_left, " %s=%s", k, v);
676 debug_attr_buffer[ sizeof( debug_attr_buffer ) - 1 ] = '\0';
685 strcpy(buffer,"*=*");
686 if ((idx=(int)g_hash_table_lookup(attr_hash, buffer)))
689 sprintf(buffer,"%s=*", k);
690 if ((idx=(int)g_hash_table_lookup(attr_hash, buffer)))
693 sprintf(buffer,"*=%s", v);
694 if ((idx=(int)g_hash_table_lookup(attr_hash, buffer)))
697 sprintf(buffer,"%s=%s", k, v);
698 if ((idx=(int)g_hash_table_lookup(attr_hash, buffer)))
714 decode_entities(char *buffer)
719 while ((pos=strchr(pos, '&'))) {
721 for (i = 0 ; i < sizeof(entities)/sizeof(struct entity); i++) {
722 len=strlen(entities[i].entity);
723 if (!strncmp(pos, entities[i].entity, len)) {
725 memmove(pos+1, pos+len, strlen(pos+len)+1);
737 char k_buffer[BUFFER_SIZE];
738 char v_buffer[BUFFER_SIZE];
739 if (!xml_get_attribute(p, "k", k_buffer, BUFFER_SIZE))
741 if (!xml_get_attribute(p, "v", v_buffer, BUFFER_SIZE))
743 decode_entities(v_buffer);
744 add_tag(k_buffer, v_buffer);
756 static struct tile_head {
764 struct tile_head *next;
780 static struct buffer node_buffer = {
785 static char** th_get_subtile( const struct tile_head* th, int idx )
787 char* subtile_ptr = NULL;
788 subtile_ptr = (char*)th + sizeof( struct tile_head ) + idx * sizeof( char *);
789 return (char**)subtile_ptr;
793 extend_buffer(struct buffer *b)
795 b->malloced+=b->malloced_step;
796 b->base=realloc(b->base, b->malloced);
797 if (b->base == NULL) {
798 fprintf(stderr,"realloc of %d bytes failed\n",b->malloced);
805 GHashTable *node_hash;
808 node_buffer_to_hash(void)
810 int i,count=node_buffer.size/sizeof(struct node_item);
811 struct node_item *ni=(struct node_item *)node_buffer.base;
812 for (i = 0 ; i < count ; i++)
813 g_hash_table_insert(node_hash, (gpointer)(ni[i].id), (gpointer)i);
816 static struct node_item *ni;
819 add_node(int id, double lat, double lon)
821 if (node_buffer.size + sizeof(struct node_item) > node_buffer.malloced)
822 extend_buffer(&node_buffer);
825 item.type=type_point_unkn;
827 town_name_attr.len=0;
829 is_in_buffer[0]='\0';
830 sprintf(debug_attr_buffer,"nodeid=%d", nodeid);
831 ni=(struct node_item *)(node_buffer.base+node_buffer.size);
837 ni->c.x=lon*6371000.0*M_PI/180;
838 ni->c.y=log(tan(M_PI_4+lat*M_PI/360))*6371000.0;
839 node_buffer.size+=sizeof(struct node_item);
841 if (ni->id > nodeid_last) {
844 fprintf(stderr,"INFO: Nodes out of sequence (new %d vs old %d), adding hash\n", ni->id, nodeid_last);
845 node_hash=g_hash_table_new(NULL, NULL);
846 node_buffer_to_hash();
849 if (!g_hash_table_lookup(node_hash, (gpointer)(ni->id)))
850 g_hash_table_insert(node_hash, (gpointer)(ni->id), (gpointer)(ni-(struct node_item *)node_buffer.base));
852 node_buffer.size-=sizeof(struct node_item);
861 char id_buffer[BUFFER_SIZE];
862 char lat_buffer[BUFFER_SIZE];
863 char lon_buffer[BUFFER_SIZE];
864 if (!xml_get_attribute(p, "id", id_buffer, BUFFER_SIZE))
866 if (!xml_get_attribute(p, "lat", lat_buffer, BUFFER_SIZE))
868 if (!xml_get_attribute(p, "lon", lon_buffer, BUFFER_SIZE))
870 add_node(atoi(id_buffer), atof(lat_buffer), atof(lon_buffer));
875 static struct node_item *
876 node_item_get(int id)
878 struct node_item *ni=(struct node_item *)(node_buffer.base);
879 int count=node_buffer.size/sizeof(struct node_item);
880 int interval=count/4;
884 i=(int)(g_hash_table_lookup(node_hash, (gpointer)id));
887 while (ni[p].id != id) {
889 fprintf(stderr,"p=%d count=%d interval=%d id=%d ni[p].id=%d\n", p, count, interval, id, ni[p].id);
924 struct node_item *ni=node_item_get(id);
926 fprintf(stderr,"WARNING: node id %d not found\n", id);
938 item.type=type_street_unkn;
940 street_name_attr.len=0;
941 street_name_systematic_attr.len=0;
944 sprintf(debug_attr_buffer,"wayid=%d", wayid);
950 char id_buffer[BUFFER_SIZE];
951 if (!xml_get_attribute(p, "id", id_buffer, BUFFER_SIZE))
953 add_way(atoi(id_buffer));
958 parse_relation(char *p)
960 debug_attr_buffer[0]='\0';
965 write_attr(FILE *out, struct attr_bin *attr, void *buffer)
968 fwrite(attr, sizeof(*attr), 1, out);
969 fwrite(buffer, (attr->len-1)*4, 1, out);
974 attr_longest_match(struct attr_mapping **mapping, int mapping_count, enum item_type *types, int types_count)
976 int i,j,longest=0,ret=0,sum,val;
977 struct attr_mapping *curr;
978 for (i = 0 ; i < mapping_count ; i++) {
981 for (j = 0 ; j < curr->attr_present_idx_count ; j++) {
982 val=attr_present[curr->attr_present_idx[j]];
994 if (sum > 0 && sum == longest && ret < types_count)
995 types[ret++]=curr->type;
997 memset(attr_present, 0, sizeof(*attr_present)*attr_present_count);
1005 enum item_type types[5];
1009 if (dedupe_ways_hash) {
1010 if (g_hash_table_lookup(dedupe_ways_hash, (gpointer)wayid))
1012 g_hash_table_insert(dedupe_ways_hash, (gpointer)wayid, (gpointer)1);
1014 count=attr_longest_match(attr_mapping_way, attr_mapping_way_count, types, sizeof(types)/sizeof(enum item_type));
1015 pad_text_attr(&debug_attr, debug_attr_buffer);
1017 alen+=label_attr.len+1;
1018 if (street_name_systematic_attr.len)
1019 alen+=street_name_systematic_attr.len+1;
1021 alen+=debug_attr.len+1;
1023 alen+=flags_attr.len+1;
1027 item.type=type_street_unkn;
1028 if (coverage && item_is_street(item))
1029 item.type=type_coverage;
1030 item.clen=coord_count*2;
1031 item.len=item.clen+2+alen;
1032 fwrite(&item, sizeof(item), 1, out);
1033 fwrite(coord_buffer, coord_count*sizeof(struct coord), 1, out);
1034 if (item_is_street(item)) {
1035 street_name_attr.len=label_attr.len;
1036 write_attr(out, &street_name_attr, label_attr_buffer);
1038 write_attr(out, &label_attr, label_attr_buffer);
1039 write_attr(out, &street_name_systematic_attr, street_name_systematic_attr_buffer);
1040 write_attr(out, &debug_attr, debug_attr_buffer);
1041 write_attr(out, &flags_attr, &flags_attr_value);
1047 int alen=0,conflict=0,count;
1048 enum item_type types[5];
1049 struct country_table *result=NULL, *lookup;
1050 if (!out || ! node_is_tagged || ! nodeid)
1052 count=attr_longest_match(attr_mapping_node, attr_mapping_node_count, types, sizeof(types)/sizeof(enum item_type));
1053 pad_text_attr(&debug_attr, debug_attr_buffer);
1055 alen+=label_attr.len+1;
1057 alen+=debug_attr.len+1;
1061 item.type=type_point_unkn;
1063 item.len=item.clen+2+alen;
1064 fwrite(&item, sizeof(item), 1, out);
1065 fwrite(&ni->c, 1*sizeof(struct coord), 1, out);
1066 if (item_is_town(item)) {
1067 town_name_attr.len=label_attr.len;
1068 write_attr(out, &town_name_attr, label_attr_buffer);
1070 write_attr(out, &label_attr, label_attr_buffer);
1071 write_attr(out, &debug_attr, debug_attr_buffer);
1072 #ifdef GENERATE_INDEX
1073 if (item_is_town(item) && town_name_attr.len) {
1074 char *tok,*buf=is_in_buffer;
1075 while ((tok=strtok(buf, ","))) {
1078 lookup=g_hash_table_lookup(country_table_hash,tok);
1080 if (result && result->countryid != lookup->countryid) {
1081 fprintf(stderr,"conflict for %s %s country %d vs %d\n", label_attr_buffer, debug_attr_buffer, lookup->countryid, result->countryid);
1088 if (result && !conflict) {
1089 if (!result->file) {
1090 char *name=g_strdup_printf("country_%d.bin.unsorted", result->countryid);
1091 result->file=fopen(name,"w");
1096 item.len=item.clen+2+label_attr.len+1;
1097 fwrite(&item, sizeof(item), 1, result->file);
1098 fwrite(&ni->c, 1*sizeof(struct coord), 1, result->file);
1099 write_attr(result->file, &town_name_attr, label_attr_buffer);
1101 result->size+=(item.clen+3+label_attr.len+1)*4;
1107 processed_nodes_out++;
1111 sort_countries_compare(const void *p1, const void *p2)
1113 struct item_bin *ib1=*((struct item_bin **)p1),*ib2=*((struct item_bin **)p2);
1114 struct attr_bin *attr1,*attr2;
1116 assert(ib1->clen==2);
1117 assert(ib2->clen==2);
1118 attr1=(struct attr_bin *)((int *)(ib1+1)+ib1->clen);
1119 attr2=(struct attr_bin *)((int *)(ib2+1)+ib1->clen);
1120 assert(attr1->type == attr_town_name);
1121 assert(attr2->type == attr_town_name);
1122 s1=(char *)(attr1+1);
1123 s2=(char *)(attr2+1);
1124 return strcmp(s1, s2);
1126 fprintf(stderr,"sort_countries_compare p1=%p p2=%p %s %s\n",p1,p2,s1,s2);
1131 #ifdef GENERATE_INDEX
1133 sort_countries(void)
1136 struct country_table *co;
1138 struct item_bin *ib;
1140 char *p,*buffer,**idx,*name;
1141 for (i = 0 ; i < sizeof(country_table)/sizeof(struct country_table) ; i++) {
1142 co=&country_table[i];
1148 buffer=malloc(co->size);
1149 assert(buffer != NULL);
1150 idx=malloc(co->count*sizeof(void *));
1151 assert(idx != NULL);
1152 name=g_strdup_printf("country_%d.bin.unsorted", co->countryid);
1155 fread(buffer, co->size, 1, f);
1160 for (j = 0 ; j < co->count ; j++) {
1162 p+=(*((int *)p)+1)*4;
1164 qsort(idx, co->count, sizeof(void *), sort_countries_compare);
1165 name=g_strdup_printf("country_%d.bin", co->countryid);
1167 for (j = 0 ; j < co->count ; j++) {
1168 ib=(struct item_bin *)(idx[j]);
1169 c=(struct coord *)(ib+1);
1170 fwrite(ib, (ib->len+1)*4, 1, f);
1172 bbox_extend(c, &co->r);
1184 add_nd(char *p, int ref)
1187 struct node_item *ni;
1188 ni=node_item_get(ref);
1191 coord_buffer[coord_count++]=ni->c;
1193 SET_REF(coord_buffer[coord_count], ref);
1199 if (len > 0 && p[len-1]=='\n')
1201 fprintf(stderr,"WARNING: way %d: node %d not found (%s)\n",wayid,ref,p);
1203 if (coord_count > 65536) {
1204 fprintf(stderr,"ERROR: Overflow\n");
1212 char ref_buffer[BUFFER_SIZE];
1213 if (!xml_get_attribute(p, "ref", ref_buffer, BUFFER_SIZE))
1215 add_nd(p, atoi(ref_buffer));
1221 save_buffer(char *filename, struct buffer *b)
1224 f=fopen(filename,"wb+");
1225 fwrite(b->base, b->size, 1, f);
1230 load_buffer(char *filename, struct buffer *b)
1236 f=fopen(filename,"rb");
1237 fseek(f, 0, SEEK_END);
1238 b->size=b->malloced=ftell(f);
1239 fprintf(stderr,"reading %d bytes from %s\n", b->size, filename);
1240 fseek(f, 0, SEEK_SET);
1241 b->base=malloc(b->size);
1242 assert(b->base != NULL);
1243 fread(b->base, b->size, 1, f);
1248 phase1(FILE *in, FILE *out_ways, FILE *out_nodes)
1254 while (fgets(buffer, size, in)) {
1255 p=strchr(buffer,'<');
1257 fprintf(stderr,"WARNING: wrong line %s\n", buffer);
1260 if (!strncmp(p, "<?xml ",6)) {
1261 } else if (!strncmp(p, "<osm ",5)) {
1262 } else if (!strncmp(p, "<bound ",7)) {
1263 } else if (!strncmp(p, "<node ",6)) {
1265 fprintf(stderr,"WARNING: failed to parse %s\n", buffer);
1268 } else if (!strncmp(p, "<tag ",5)) {
1270 fprintf(stderr,"WARNING: failed to parse %s\n", buffer);
1271 } else if (!strncmp(p, "<way ",5)) {
1274 fprintf(stderr,"WARNING: failed to parse %s\n", buffer);
1276 } else if (!strncmp(p, "<nd ",4)) {
1278 fprintf(stderr,"WARNING: failed to parse %s\n", buffer);
1279 } else if (!strncmp(p, "<relation ",10)) {
1281 if (!parse_relation(p))
1282 fprintf(stderr,"WARNING: failed to parse %s\n", buffer);
1283 processed_relations++;
1284 } else if (!strncmp(p, "<member ",8)) {
1285 } else if (!strncmp(p, "</node>",7)) {
1287 end_node(out_nodes);
1288 } else if (!strncmp(p, "</way>",6)) {
1291 } else if (!strncmp(p, "</relation>",11)) {
1293 } else if (!strncmp(p, "</osm>",6)) {
1295 fprintf(stderr,"WARNING: unknown tag in %s\n", buffer);
1303 #ifdef HAVE_POSTGRESQL
1305 phase1_db(char *dbstr, FILE *out_ways, FILE *out_nodes)
1308 PGresult *res,*node,*way,*tag;
1309 int count,tagged,i,j,k;
1310 long min, max, id, tag_id, node_id;
1314 conn=PQconnectdb(dbstr);
1316 fprintf(stderr,"Failed to connect to database with '%s'\n",dbstr);
1319 res=PQexec(conn, "begin");
1321 fprintf(stderr, "Cannot begin transaction: %s\n", PQerrorMessage(conn));
1325 res=PQexec(conn, "set transaction isolation level serializable");
1327 fprintf(stderr, "Cannot set isolation level: %s\n", PQerrorMessage(conn));
1331 res=PQexec(conn, "declare node cursor for select id,x(coordinate),y(coordinate) from node order by id");
1333 fprintf(stderr, "Cannot setup cursor for nodes: %s\n", PQerrorMessage(conn));
1337 res=PQexec(conn, "declare way cursor for select id from way order by id");
1339 fprintf(stderr, "Cannot setup cursor for nodes: %s\n", PQerrorMessage(conn));
1343 res=PQexec(conn, "declare relation cursor for select id from relation order by id");
1345 fprintf(stderr, "Cannot setup cursor for nodes: %s\n", PQerrorMessage(conn));
1350 node=PQexec(conn, "fetch 100000 from node");
1352 fprintf(stderr, "Cannot setup cursor for nodes: %s\n", PQerrorMessage(conn));
1356 count=PQntuples(node);
1359 min=atol(PQgetvalue(node, 0, 0));
1360 max=atol(PQgetvalue(node, count-1, 0));
1361 sprintf(query,"select node_id,name,value from node_tag where node_id >= %ld and node_id <= %ld order by node_id", min, max);
1362 tag=PQexec(conn, query);
1364 fprintf(stderr, "Cannot query node_tag: %s\n", PQerrorMessage(conn));
1368 for (i = 0 ; i < count ; i++) {
1369 id=atol(PQgetvalue(node, i, 0));
1370 add_node(id, atof(PQgetvalue(node, i, 1)), atof(PQgetvalue(node, i, 2)));
1374 while (j < PQntuples(tag)) {
1375 tag_id=atol(PQgetvalue(tag, j, 0));
1377 add_tag(PQgetvalue(tag, j, 1), PQgetvalue(tag, j, 2));
1387 end_node(out_nodes);
1394 way=PQexec(conn, "fetch 100000 from way");
1396 fprintf(stderr, "Cannot setup cursor for ways: %s\n", PQerrorMessage(conn));
1400 count=PQntuples(way);
1403 min=atol(PQgetvalue(way, 0, 0));
1404 max=atol(PQgetvalue(way, count-1, 0));
1405 sprintf(query,"select way_id,node_id from way_node where way_id >= %ld and way_id <= %ld order by way_id,sequence_id", min, max);
1406 node=PQexec(conn, query);
1408 fprintf(stderr, "Cannot query way_node: %s\n", PQerrorMessage(conn));
1411 sprintf(query,"select way_id,name,value from way_tag where way_id >= %ld and way_id <= %ld order by way_id", min, max);
1412 tag=PQexec(conn, query);
1414 fprintf(stderr, "Cannot query way_tag: %s\n", PQerrorMessage(conn));
1419 for (i = 0 ; i < count ; i++) {
1420 id=atol(PQgetvalue(way, i, 0));
1425 while (k < PQntuples(node)) {
1426 node_id=atol(PQgetvalue(node, k, 0));
1427 if (node_id == id) {
1428 add_nd("",atol(PQgetvalue(node, k, 1)));
1437 while (j < PQntuples(tag)) {
1438 tag_id=atol(PQgetvalue(tag, j, 0));
1440 add_tag(PQgetvalue(tag, j, 1), PQgetvalue(tag, j, 2));
1458 res=PQexec(conn, "commit");
1460 fprintf(stderr, "Cannot commit transaction: %s\n", PQerrorMessage(conn));
1470 static char buffer[150000];
1474 static struct item_bin *
1477 struct item_bin *ib=(struct item_bin *) buffer;
1479 r=fread(ib, sizeof(*ib), 1, in);
1483 assert((ib->len+1) < sizeof(buffer));
1484 s=(ib->len+1)*4-sizeof(*ib);
1485 r=fread(ib+1, s, 1, in);
1493 bbox_extend(struct coord *c, struct rect *r)
1506 bbox(struct coord *c, int count, struct rect *r)
1519 contains_bbox(int xl, int yl, int xh, int yh, struct rect *r)
1521 if (r->h.x < xl || r->h.x > xh) {
1524 if (r->l.x > xh || r->l.x < xl) {
1527 if (r->h.y < yl || r->h.y > yh) {
1530 if (r->l.y > yh || r->l.y < yl) {
1536 struct rect world_bbox = {
1537 { -20000000, -20000000},
1538 { 20000000, 20000000},
1542 tile(struct rect *r, char *ret, int max)
1551 for (i = 0 ; i < max ; i++) {
1558 if ( contains_bbox(x0,y0,x2,y2,r)) {
1562 } else if (contains_bbox(x2,y0,x4,y2,r)) {
1566 } else if (contains_bbox(x0,y2,x2,y4,r)) {
1570 } else if (contains_bbox(x2,y2,x4,y4,r)) {
1580 tile_bbox(char *tile, struct rect *r)
1585 c.x=(r->l.x+r->h.x)/2;
1586 c.y=(r->l.y+r->h.y)/2;
1609 GHashTable *tile_hash;
1610 GHashTable *tile_hash2;
1613 tile_extend(char *tile, struct item_bin *ib, GList **tiles_list)
1615 struct tile_head *th=NULL;
1616 if (debug_tile(tile))
1617 fprintf(stderr,"Tile:Writing %d bytes to '%s' (%p,%p)\n", (ib->len+1)*4, tile, g_hash_table_lookup(tile_hash, tile), tile_hash2 ? g_hash_table_lookup(tile_hash2, tile) : NULL);
1619 th=g_hash_table_lookup(tile_hash2, tile);
1621 th=g_hash_table_lookup(tile_hash, tile);
1623 th=malloc(sizeof(struct tile_head)+ sizeof( char* ) );
1625 // strcpy(th->subtiles, tile);
1628 th->total_size_used=0;
1631 th->name=string_hash_lookup(tile);
1632 *th_get_subtile( th, 0 ) = th->name;
1635 g_hash_table_insert(tile_hash2, string_hash_lookup( th->name ), th);
1637 *tiles_list=g_list_append(*tiles_list, string_hash_lookup( th->name ) );
1639 if (debug_tile(tile))
1640 fprintf(stderr,"new '%s'\n", tile);
1642 th->total_size+=ib->len*4+4;
1643 if (debug_tile(tile))
1644 fprintf(stderr,"New total size of %s(%p):%d\n", th->name, th, th->total_size);
1645 g_hash_table_insert(tile_hash, string_hash_lookup( th->name ), th);
1649 tile_data_size(char *tile)
1651 struct tile_head *th;
1652 th=g_hash_table_lookup(tile_hash, tile);
1655 return th->total_size;
1659 merge_tile(char *base, char *sub)
1661 struct tile_head *thb, *ths;
1662 thb=g_hash_table_lookup(tile_hash, base);
1663 ths=g_hash_table_lookup(tile_hash, sub);
1666 if (debug_tile(base) || debug_tile(sub))
1667 fprintf(stderr,"merging '%s'(%p) (%d) with '%s'(%p) (%d)\n", base, thb, thb ? thb->total_size : 0, sub, ths, ths->total_size);
1670 g_hash_table_remove(tile_hash, sub);
1671 thb->name=string_hash_lookup(base);
1672 g_hash_table_insert(tile_hash, string_hash_lookup( thb->name ), thb);
1675 thb=realloc(thb, sizeof(struct tile_head)+( ths->num_subtiles+thb->num_subtiles ) * sizeof( char*) );
1676 assert(thb != NULL);
1677 memcpy( th_get_subtile( thb, thb->num_subtiles ), th_get_subtile( ths, 0 ), ths->num_subtiles * sizeof( char*) );
1678 thb->num_subtiles+=ths->num_subtiles;
1679 thb->total_size+=ths->total_size;
1680 g_hash_table_insert(tile_hash, string_hash_lookup( thb->name ), thb);
1681 g_hash_table_remove(tile_hash, sub);
1689 get_tiles_list_func(char *key, struct tile_head *th, GList **list)
1691 *list=g_list_prepend(*list, key);
1695 get_tiles_list(void)
1698 g_hash_table_foreach(tile_hash, (GHFunc)get_tiles_list_func, &ret);
1704 write_tile(char *key, struct tile_head *th, gpointer dummy)
1708 fprintf(stderr,"DEBUG: Writing %s\n", key);
1709 strcpy(buffer,"tiles/");
1712 strcat(buffer,".bin");
1714 f=fopen(buffer, "wb+");
1716 fwrite(th->data, th->size, 1, f);
1724 write_item(char *tile, struct item_bin *ib)
1726 struct tile_head *th;
1729 th=g_hash_table_lookup(tile_hash2, tile);
1731 th=g_hash_table_lookup(tile_hash, tile);
1733 if (th->process != 0 && th->process != 1) {
1734 fprintf(stderr,"error with tile '%s' of length %d\n", tile, strlen(tile));
1739 if (debug_tile(tile))
1740 fprintf(stderr,"Data:Writing %d bytes to '%s' (%p,%p)\n", (ib->len+1)*4, tile, g_hash_table_lookup(tile_hash, tile), tile_hash2 ? g_hash_table_lookup(tile_hash2, tile) : NULL);
1742 if (th->total_size_used+size > th->total_size) {
1743 fprintf(stderr,"Overflow in tile %s (used %d max %d item %d)\n", tile, th->total_size_used, th->total_size, size);
1747 memcpy(th->zip_data+th->total_size_used, ib, size);
1748 th->total_size_used+=size;
1750 fprintf(stderr,"no tile hash found for %s\n", tile);
1757 write_item_part(FILE *out, struct item_bin *orig, int first, int last)
1759 struct item_bin new;
1760 struct coord *c=(struct coord *)(orig+1);
1761 char *attr=(char *)(c+orig->clen/2);
1762 int attr_len=orig->len-orig->clen-2;
1764 new.type=orig->type;
1765 new.clen=(last-first+1)*2;
1766 new.len=new.clen+attr_len+2;
1768 fprintf(stderr,"first %d last %d type 0x%x len %d clen %d attr_len %d\n", first, last, new.type, new.len, new.clen, attr_len);
1770 fwrite(&new, sizeof(new), 1, out);
1771 fwrite(c+first, new.clen*4, 1, out);
1772 fwrite(attr, attr_len*4, 1, out);
1776 phase2(FILE *in, FILE *out)
1779 int i,ccount,last,ndref;
1780 struct item_bin *ib;
1781 struct node_item *ni;
1783 processed_nodes=processed_nodes_out=processed_ways=processed_relations=processed_tiles=0;
1785 while ((ib=read_item(in))) {
1787 fprintf(stderr,"type 0x%x len %d clen %d\n", ib->type, ib->len, ib->clen);
1790 c=(struct coord *)(ib+1);
1792 for (i = 0 ; i < ccount ; i++) {
1795 ni=node_item_get(ndref);
1797 fprintf(stderr,"ni=%p\n", ni);
1800 if (ni->ref_way > 1 && i != 0 && i != ccount-1 && item_is_street(*ib)) {
1801 write_item_part(out, ib, last, i);
1806 write_item_part(out, ib, last, ccount-1);
1814 phase34_process_file(int phase, FILE *in)
1816 struct item_bin *ib;
1821 while ((ib=read_item(in))) {
1822 if (ib->type < 0x80000000)
1826 bbox((struct coord *)(ib+1), ib->clen/2, &r);
1829 if (ib->type == type_street_n_lanes || ib->type == type_highway_city || ib->type == type_highway_land || ib->type == type_ramp)
1831 if (ib->type == type_street_3_city || ib->type == type_street_4_city || ib->type == type_street_3_land || ib->type == type_street_4_land)
1834 tile(&r, buffer, max);
1836 fprintf(stderr,"%s\n", buffer);
1839 tile_extend(buffer, ib, NULL);
1841 write_item(buffer, ib);
1846 struct item_bin item;
1848 struct attr_bin attr_order;
1851 struct attr_bin attr_zipfile_ref;
1856 index_submap_add(int phase, struct tile_head *th, GList **tiles_list)
1858 struct index_item ii;
1859 int len=strlen(th->name);
1860 char index_tile[len+1];
1862 ii.min=(len > 4) ? len-4 : 0;
1864 strcpy(index_tile, th->name);
1870 tile_bbox(th->name, &ii.r);
1872 ii.item.len=sizeof(ii)/4-1;
1873 ii.item.type=type_submap;
1876 ii.attr_order.len=2;
1877 ii.attr_order.type=attr_order;
1879 ii.attr_zipfile_ref.len=2;
1880 ii.attr_zipfile_ref.type=attr_zipfile_ref;
1881 ii.zipfile_ref=th->zipnum;
1884 tile_extend(index_tile, (struct item_bin *)&ii, tiles_list);
1886 write_item(index_tile, (struct item_bin *)&ii);
1888 unsigned int *c=(unsigned int *)ⅈ
1890 for (i = 0 ; i < sizeof(ii)/4 ; i++) {
1891 fprintf(stderr,"%08x ", c[i]);
1893 fprintf(stderr,"\n");
1898 add_tile_hash(struct tile_head *th)
1900 int idx,len,maxnamelen=0;
1904 g_hash_table_insert(tile_hash2, string_hash_lookup( th->name ), th);
1906 for( idx = 0; idx < th->num_subtiles; idx++ ) {
1908 data = th_get_subtile( th, idx );
1910 if (debug_tile(data) || debug_tile(th->name)) {
1911 fprintf(stderr,"Parent for '%s' is '%s'\n", *data, th->name);
1914 g_hash_table_insert(tile_hash2, *data, th);
1916 len = strlen( *data );
1918 if (len > maxnamelen) {
1927 create_tile_hash(void)
1929 struct tile_head *th;
1930 int len,maxnamelen=0;
1932 tile_hash2=g_hash_table_new(g_str_hash, g_str_equal);
1935 len=add_tile_hash(th);
1936 if (len > maxnamelen)
1944 create_tile_hash_list(GList *list)
1947 struct tile_head *th;
1949 tile_hash2=g_hash_table_new(g_str_hash, g_str_equal);
1951 fprintf(stderr,"list=%p\n", list);
1952 next=g_list_first(list);
1954 th=g_hash_table_lookup(tile_hash, next->data);
1956 fprintf(stderr,"No tile found for '%s'\n", (char *)(next->data));
1959 next=g_list_next(next);
1964 destroy_tile_hash(void)
1966 g_hash_table_destroy(tile_hash2);
1972 static void write_countrydir(int phase, int maxnamelen);
1975 write_tilesdir(int phase, int maxlen, FILE *out)
1978 GList *tiles_list,*next;
1980 struct tile_head *th,**last=NULL;
1983 tiles_list=get_tiles_list();
1985 create_tile_hash_list(tiles_list);
1986 next=g_list_first(tiles_list);
1987 last=&tile_head_root;
1990 if (strlen(next->data) > maxlen)
1991 maxlen=strlen(next->data);
1992 next=g_list_next(next);
1998 fprintf(stderr,"PROGRESS: collecting tiles with len=%d\n", len);
2000 #ifdef GENERATE_INDEX
2002 write_countrydir(phase, maxlen);
2004 next=g_list_first(tiles_list);
2006 if (strlen(next->data) == len) {
2007 th=g_hash_table_lookup(tile_hash, next->data);
2013 fprintf(out,"%s:%d",(char *)next->data,th->total_size);
2015 for ( idx = 0; idx< th->num_subtiles; idx++ ){
2016 data= th_get_subtile( th, idx );
2017 fprintf(out,":%s", *data);
2023 index_submap_add(phase, th, &tiles_list);
2027 next=g_list_next(next);
2036 struct tile_head *th;
2037 char basetile[1024];
2039 GList *tiles_list_sorted,*last;
2040 int i,i_min,len,size_all,size[5],size_min,work_done;
2044 tiles_list_sorted=get_tiles_list();
2045 fprintf(stderr,"PROGRESS: sorting %d tiles\n", g_list_length(tiles_list_sorted));
2046 tiles_list_sorted=g_list_sort(tiles_list_sorted, (GCompareFunc)strcmp);
2047 fprintf(stderr,"PROGRESS: sorting %d tiles done\n", g_list_length(tiles_list_sorted));
2048 last=g_list_last(tiles_list_sorted);
2051 th=g_hash_table_lookup(tile_hash, last->data);
2052 zip_size+=th->total_size;
2053 last=g_list_previous(last);
2055 fprintf(stderr,"DEBUG: size=%Ld\n", zip_size);
2056 last=g_list_last(tiles_list_sorted);
2060 len=strlen(last->data);
2062 strcpy(basetile,last->data);
2063 basetile[len-1]='\0';
2064 strcpy(subtile,last->data);
2065 for (i = 0 ; i < 4 ; i++) {
2066 subtile[len-1]='a'+i;
2067 size[i]=tile_data_size(subtile);
2069 size[4]=tile_data_size(basetile);
2070 size_all=size[0]+size[1]+size[2]+size[3]+size[4];
2071 if (size_all < 65536 && size_all > 0 && size_all != size[4]) {
2072 for (i = 0 ; i < 4 ; i++) {
2073 subtile[len-1]='a'+i;
2074 work_done+=merge_tile(basetile, subtile);
2080 for (i = 0 ; i < 4 ; i++) {
2081 if (size[i] && size[i] < size_min) {
2088 if (size[4]+size_min >= 65536)
2090 subtile[len-1]='a'+i_min;
2091 work_done+=merge_tile(basetile, subtile);
2092 size[4]+=size[i_min];
2097 last=g_list_previous(last);
2099 g_list_free(tiles_list_sorted);
2100 fprintf(stderr,"PROGRESS: merged %d tiles\n", work_done);
2101 } while (work_done);
2104 struct country_index_item {
2105 struct item_bin item;
2106 struct attr_bin attr_country_id;
2108 struct attr_bin attr_zipfile_ref;
2113 index_country_add(int phase, int country_id, int zipnum)
2115 struct country_index_item ii;
2116 char *index_tile="";
2118 ii.item.len=sizeof(ii)/4-1;
2119 ii.item.type=type_countryindex;
2122 ii.attr_country_id.len=2;
2123 ii.attr_country_id.type=attr_country_id;
2124 ii.country_id=country_id;
2126 ii.attr_zipfile_ref.len=2;
2127 ii.attr_zipfile_ref.type=attr_zipfile_ref;
2128 ii.zipfile_ref=zipnum;
2131 tile_extend(index_tile, (struct item_bin *)&ii, NULL);
2133 write_item(index_tile, (struct item_bin *)&ii);
2136 #ifdef GENERATE_INDEX
2144 add_aux_tile(int phase, char *name, char *filename, int size)
2146 struct aux_tile *at;
2148 at=g_new0(struct aux_tile, 1);
2149 at->name=g_strdup(name);
2150 at->filename=g_strdup(filename);
2152 aux_tile_list=g_list_append(aux_tile_list, at);
2158 write_aux_tiles(FILE *out, FILE *dir_out, int compression_level, int namelen)
2160 GList *l=aux_tile_list;
2161 struct aux_tile *at;
2168 buffer=malloc(at->size);
2169 assert(buffer != NULL);
2170 f=fopen(at->filename,"r");
2172 fread(buffer, at->size, 1, f);
2174 write_zipmember(out, dir_out, at->name, namelen, buffer, at->size, compression_level);
2182 write_countrydir(int phase, int maxnamelen)
2187 char searchtile[32];
2189 struct country_table *co;
2190 for (i = 0 ; i < sizeof(country_table)/sizeof(struct country_table) ; i++) {
2191 co=&country_table[i];
2194 tile(&co->r, tilename, max);
2195 sprintf(searchtile,"%ss%d", tilename, 0);
2196 sprintf(filename,"country_%d.bin", co->countryid);
2197 zipnum=add_aux_tile(phase, searchtile, filename, co->size);
2198 index_country_add(phase,co->countryid,zipnum);
2204 remove_countryfiles(void)
2208 struct country_table *co;
2210 for (i = 0 ; i < sizeof(country_table)/sizeof(struct country_table) ; i++) {
2211 co=&country_table[i];
2213 sprintf(filename,"country_%d.bin", co->countryid);
2221 phase34(int phase, int maxnamelen, FILE *ways_in, FILE *nodes_in, FILE *tilesdir_out)
2224 processed_nodes=processed_nodes_out=processed_ways=processed_relations=processed_tiles=0;
2228 tile_hash=g_hash_table_new(g_str_hash, g_str_equal);
2230 phase34_process_file(phase, ways_in);
2232 phase34_process_file(phase, nodes_in);
2233 fprintf(stderr,"read %d bytes\n", bytes_read);
2238 write_tilesdir(phase, maxnamelen, tilesdir_out);
2245 dump_coord(struct coord *c)
2247 printf("0x%x 0x%x",c->x, c->y);
2253 struct item_bin *ib;
2261 while ((ib=read_item(in))) {
2262 c=(struct coord *)(ib+1);
2263 if (ib->type < type_line) {
2267 attr_start=(int *)(ib+1)+ib->clen;
2268 attr_end=(int *)ib+ib->len+1;
2269 printf("type=%s", item_to_name(ib->type));
2270 while (attr_start < attr_end) {
2271 a=(struct attr_bin *)(attr_start);
2272 attr_start+=a->len+1;
2274 attr_data_set(&attr, (a+1));
2275 str=attr_to_text(&attr, NULL, 1);
2276 printf(" %s=\"%s\"", attr_to_name(a->type), str);
2279 printf(" debug=\"length=%d\"", ib->len);
2281 if (ib->type >= type_line) {
2282 for (i = 0 ; i < ib->clen/2 ; i++) {
2292 phase3(FILE *ways_in, FILE *nodes_in, FILE *tilesdir_out)
2294 return phase34(3, 0, ways_in, nodes_in, tilesdir_out);
2297 static long long zipoffset;
2298 static int zipdir_size;
2301 compress2_int(Byte *dest, uLongf *destLen, const Bytef *source, uLong sourceLen, int level)
2306 stream.next_in = (Bytef*)source;
2307 stream.avail_in = (uInt)sourceLen;
2308 stream.next_out = dest;
2309 stream.avail_out = (uInt)*destLen;
2310 if ((uLong)stream.avail_out != *destLen) return Z_BUF_ERROR;
2312 stream.zalloc = (alloc_func)0;
2313 stream.zfree = (free_func)0;
2314 stream.opaque = (voidpf)0;
2316 err = deflateInit2(&stream, level, Z_DEFLATED, -15, 9, Z_DEFAULT_STRATEGY);
2317 if (err != Z_OK) return err;
2319 err = deflate(&stream, Z_FINISH);
2320 if (err != Z_STREAM_END) {
2321 deflateEnd(&stream);
2322 return err == Z_OK ? Z_BUF_ERROR : err;
2324 *destLen = stream.total_out;
2326 err = deflateEnd(&stream);
2331 write_zipmember(FILE *out, FILE *dir_out, char *name, int filelen, char *data, int data_size, int compression_level)
2333 struct zip_lfh lfh = {
2346 struct zip_cd cd = {
2367 char filename[filelen+1];
2368 int error,crc,len,comp_size=data_size;
2369 uLongf destlen=data_size+data_size/500+12;
2370 char compbuffer[destlen];
2372 crc=crc32(0, NULL, 0);
2373 crc=crc32(crc, (unsigned char *)data, data_size);
2374 if (compression_level) {
2375 error=compress2_int((Byte *)compbuffer, &destlen, (Bytef *)data, data_size, compression_level);
2376 if (error == Z_OK) {
2377 if (destlen < data_size) {
2382 fprintf(stderr,"compress2 returned %d\n", error);
2386 lfh.zipsize=comp_size;
2387 lfh.zipuncmp=data_size;
2388 lfh.zipmthd=compression_level ? 8:0;
2390 cd.zipcsiz=comp_size;
2391 cd.zipcunc=data_size;
2392 cd.zipcmthd=compression_level ? 8:0;
2393 strcpy(filename, name);
2394 len=strlen(filename);
2395 while (len < filelen) {
2396 filename[len++]='_';
2398 filename[filelen]='\0';
2399 fwrite(&lfh, sizeof(lfh), 1, out);
2400 fwrite(filename, filelen, 1, out);
2401 fwrite(data, comp_size, 1, out);
2402 zipoffset+=sizeof(lfh)+filelen+comp_size;
2403 fwrite(&cd, sizeof(cd), 1, dir_out);
2404 fwrite(filename, filelen, 1, dir_out);
2405 zipdir_size+=sizeof(cd)+filelen;
2409 process_slice(FILE *ways_in, FILE *nodes_in, int size, int maxnamelen, FILE *out, FILE *dir_out, int compression_level)
2411 struct tile_head *th;
2412 char *slice_data,*zip_data;
2415 slice_data=malloc(size);
2416 assert(slice_data != NULL);
2417 zip_data=slice_data;
2421 th->zip_data=zip_data;
2422 zip_data+=th->total_size;
2427 fseek(ways_in, 0, SEEK_SET);
2429 fseek(nodes_in, 0, SEEK_SET);
2430 phase34(4, maxnamelen, ways_in, nodes_in, NULL);
2435 #ifdef GENERATE_INDEX
2436 if (! strlen(th->name))
2437 zipfiles+=write_aux_tiles(out, dir_out, compression_level, maxnamelen);
2439 if (th->total_size != th->total_size_used) {
2440 fprintf(stderr,"Size error '%s': %d vs %d\n", th->name, th->total_size, th->total_size_used);
2443 if (strlen(th->name))
2444 write_zipmember(out, dir_out, th->name, maxnamelen, th->zip_data, th->total_size, compression_level);
2446 write_zipmember(out, dir_out, "index", sizeof("index")-1, th->zip_data, th->total_size, compression_level);
2459 cat(FILE *in, FILE *out)
2463 while ((size=fread(buffer, 1, 4096, in)))
2464 fwrite(buffer, 1, size, out);
2468 phase4(FILE *ways_in, FILE *nodes_in, FILE *out, FILE *dir_out, int compression_level)
2470 int slice_size=1024*1024*1024;
2471 int maxnamelen,size,slices;
2473 struct tile_head *th,*th2;
2474 struct zip_eoc eoc = {
2485 maxnamelen=create_tile_hash();
2490 fprintf(stderr, "Maximum slice size %d\n", slice_size);
2492 if (size + th->total_size > slice_size) {
2493 fprintf(stderr,"Slice %d is of size %d\n", slices, size);
2497 size+=th->total_size;
2501 fprintf(stderr,"Slice %d is of size %d\n", slices, size);
2512 while (th && size+th->total_size < slice_size) {
2513 size+=th->total_size;
2517 zipfiles+=process_slice(ways_in, nodes_in, size, maxnamelen, out, dir_out, compression_level);
2520 fseek(dir_out, 0, SEEK_SET);
2522 eoc.zipenum=zipfiles;
2523 eoc.zipecenn=zipfiles;
2524 eoc.zipecsz=zipdir_size;
2525 eoc.zipeofst=zipoffset;
2526 fwrite(&eoc, sizeof(eoc), 1, out);
2536 fprintf(f,"osm2navit - parse osm textfile and converts to NavIt binfile format\n\n");
2537 fprintf(f,"Usage :\n");
2538 fprintf(f,"bzcat planet.osm.bz2 | osm2navit mymap.bin\n");
2539 fprintf(f,"Available switches:\n");
2540 fprintf(f,"-h (--help) : this screen\n");
2541 fprintf(f,"-N (--nodes-only) : process only nodes\n");
2542 fprintf(f,"-W (--ways-only) : process only ways\n");
2543 fprintf(f,"-a (--attr-debug-level) : control which data is included in the debug attribute\n");
2544 fprintf(f,"-c (--dump-coordinates) : dump coordinates after phase 1\n");
2545 #ifdef HAVE_POSTGRESQL
2546 fprintf(f,"-d (--db) : get osm data out of a postgresql database with osm simple scheme and given connect string\n");
2548 fprintf(f,"-e (--end) : end at specified phase\n");
2549 fprintf(f,"-k (--keep-tmpfiles) : do not delete tmp files after processing. useful to reuse them\n\n");
2550 fprintf(f,"-o (--coverage) : map every street to item overage\n");
2551 fprintf(f,"-s (--start) : start at specified phase\n");
2552 fprintf(f,"-i (--input-file) : specify the input file name (OSM), overrules default stdin\n");
2553 fprintf(f,"-w (--dedupe-ways) : ensure no duplicate ways or nodes. useful when using several input files\n");
2554 fprintf(f,"-z (--compression-level) : set the compression level\n");
2559 process_binfile(FILE *in, FILE *out)
2561 struct item_bin *ib;
2562 while ((ib=read_item(in))) {
2563 fwrite(ib, (ib->len+1)*4, 1, out);
2567 int main(int argc, char **argv)
2569 FILE *ways=NULL,*ways_split=NULL,*nodes=NULL,*tilesdir,*zipdir,*res;
2570 char *map=g_strdup(attrmap);
2571 int i,c,start=1,end=4,dump_coordinates=0;
2572 int keep_tmpfiles=0;
2573 int process_nodes=1, process_ways=1;
2574 int compression_level=9;
2577 char *result,*dbstr=NULL;
2578 FILE* input_file = stdin;
2579 struct plugins *plugins=NULL;
2580 struct attr **attrs;
2584 int this_option_optind = optind ? optind : 1;
2586 int option_index = 0;
2587 static struct option long_options[] = {
2588 {"attr-debug-level", 1, 0, 'a'},
2589 {"binfile", 0, 0, 'b'},
2590 {"compression-level", 1, 0, 'z'},
2591 {"coverage", 0, 0, 'o'},
2592 #ifdef HAVE_POSTGRESQL
2595 {"dedupe-ways", 0, 0, 'w'},
2596 {"dump", 0, 0, 'D'},
2598 {"help", 0, 0, 'h'},
2599 {"keep-tmpfiles", 0, 0, 'k'},
2600 {"nodes-only", 0, 0, 'N'},
2602 {"plugin", 1, 0, 'p'},
2603 {"start", 1, 0, 's'},
2604 {"input-file", 1, 0, 'i'},
2605 {"ignore-unknown", 0, 0, 'n'},
2606 {"ways-only", 0, 0, 'W'},
2609 c = getopt_long (argc, argv, "DNWa:bc"
2610 #ifdef HAVE_POSTGRESQL
2613 "e:hi:knm:ps:w", long_options, &option_index);
2627 attr_debug_level=atoi(optarg);
2635 #ifdef HAVE_POSTGRESQL
2647 attrs=(struct attr*[]){
2648 &(struct attr){attr_type,{"textfile"}},
2649 &(struct attr){attr_data,{"bookmark.txt"}},
2652 fprintf(stderr,"optarg=%s\n", optarg);
2655 fprintf(stderr,"I will IGNORE unknown types\n");
2659 fprintf(stderr,"I will KEEP tmp files\n");
2667 plugins=plugins_new();
2668 attrs=(struct attr*[]){&(struct attr){attr_path,{optarg}},NULL};
2669 plugins_add_path(plugins, attrs);
2675 dedupe_ways_hash=g_hash_table_new(NULL, NULL);
2678 input_file = fopen( optarg, "r" );
2679 if ( input_file == NULL )
2681 fprintf( stderr, "\nInput file (%s) not found\n", optarg );
2686 compression_level=atoi(optarg);
2692 fprintf(stderr,"c=%d\n", c);
2696 if (optind != argc-(output == 1 ? 0:1))
2699 plugins_init(plugins);
2700 result=argv[optind];
2702 #ifdef GENERATE_INDEX
2703 build_countrytable();
2710 ways=fopen("ways.tmp","wb+");
2712 nodes=fopen("nodes.tmp","wb+");
2714 fprintf(stderr,"PROGRESS: Phase 1: collecting data\n");
2715 #ifdef HAVE_POSTGRESQL
2717 phase1_db(dbstr,ways,nodes);
2720 phase1(input_file,ways,nodes);
2725 #ifdef GENERATE_INDEX
2726 fprintf(stderr,"PROGRESS: Phase 1: sorting countries\n");
2730 if (end == 1 || dump_coordinates)
2731 save_buffer("coords.tmp",&node_buffer);
2735 load_buffer("coords.tmp",&node_buffer);
2738 ways=fopen("ways.tmp","rb");
2739 ways_split=fopen("ways_split.tmp","wb+");
2741 fprintf(stderr,"PROGRESS: Phase 2: finding intersections\n");
2742 phase2(ways,ways_split);
2748 fprintf(stderr,"PROGRESS: Skipping Phase 2\n");
2750 free(node_buffer.base);
2751 node_buffer.base=NULL;
2752 node_buffer.malloced=0;
2757 ways_split=fopen("ways_split.tmp","wb+");
2758 process_binfile(stdin, ways_split);
2762 fprintf(stderr,"PROGRESS: Phase 3: dumping\n");
2763 if (process_nodes) {
2764 nodes=fopen("nodes.tmp","rb");
2771 ways_split=fopen("ways_split.tmp","rb");
2781 fprintf(stderr,"PROGRESS: Phase 3: generating tiles\n");
2783 ways_split=fopen("ways_split.tmp","rb");
2785 nodes=fopen("nodes.tmp","rb");
2786 tilesdir=fopen("tilesdir.tmp","wb+");
2787 phase3(ways_split,nodes,tilesdir);
2798 fprintf(stderr,"PROGRESS: Phase 4: assembling map\n");
2800 ways_split=fopen("ways_split.tmp","rb");
2802 nodes=fopen("nodes.tmp","rb");
2803 res=fopen(result,"wb+");
2804 zipdir=fopen("zipdir.tmp","wb+");
2805 phase4(ways_split,nodes,res,zipdir,compression_level);
2812 if(!keep_tmpfiles) {
2813 remove("nodes.tmp");
2814 remove("ways_split.tmp");
2815 remove("tilesdir.tmp");
2816 remove("zipdir.tmp");
2817 #ifdef GENERATE_INDEX
2818 remove_countryfiles();