vcs.maemo.org Git - navit-package/blob - navit/osm2navit.c

   1 /**
   2  * Navit, a modular navigation system.
   3  * Copyright (C) 2005-2008 Navit Team
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License
   7  * version 2 as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write to the
  16  * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  17  * Boston, MA  02110-1301, USA.
  18  */
  19
  20
  21 #define _FILE_OFFSET_BITS 64
  22 #define _LARGEFILE_SOURCE
  23 #define _LARGEFILE64_SOURCE
  24 #include <glib.h>
  25 #include <assert.h>
  26 #include <string.h>
  27 #include <signal.h>
  28 #include <stdlib.h>
  29 #include <stdio.h>
  30 #include <math.h>
  31 #include <getopt.h>
  32 #include <unistd.h>
  33 #include <fcntl.h>
  34 #include <sys/stat.h>
  35 #include <zlib.h>
  36 #include "item.h"
  37 #include "zipfile.h"
  38 #include "config.h"
  39 #ifdef HAVE_POSTGRESQL
  40 #include <libpq-fe.h>
  41 #endif
  42
  43 #define BUFFER_SIZE 1280
  44
  45 #define GENERATE_INDEX
  46
  47 #if 1
  48 #define debug_tile(x) 0
  49 #else
  50 #define debug_tile(x) (!strcmp(x,"bcdbd") || !strcmp(x,"bcdbd") || !strcmp(x,"bcdbda") || !strcmp(x,"bcdbdb") || !strcmp(x,"bcdbdba") || !strcmp(x,"bcdbdbb") || !strcmp(x,"bcdbdbba") || !strcmp(x,"bcdbdbaa") || !strcmp(x,"bcdbdbacaa") || !strcmp(x,"bcdbdbacab") || !strcmp(x,"bcdbdbacaba") || !strcmp(x,"bcdbdbacabaa") || !strcmp(x,"bcdbdbacabab") || !strcmp(x,"bcdbdbacababb") || !strcmp(x,"bcdbdbacababba") || !strcmp(x,"bcdbdbacababbb") || !strcmp(x,"bcdbdbacababbd") || !strcmp(x,"bcdbdbacababaa") || !strcmp(x,"bcdbdbacababab") || !strcmp(x,"bcdbdbacababac") || !strcmp(x,"bcdbdbacababad") || !strcmp(x,"bcdbdbacabaaa") || !strcmp(x,"bcdbdbacabaaba") || !strcmp(x,"bcdbdbacabaabb") || !strcmp(x,"bcdbdbacabaabc") || !strcmp(x,"bcdbdbacabaabd") || !strcmp(x,"bcdbdbacabaaaa") || !strcmp(x,"bcdbdbacabaaab") || !strcmp(x,"bcdbdbacabaaac") || !strcmp(x,"bcdbdbacabaaad") || 0)
  51 #endif
  52
  53
  54 static GHashTable *dedupe_ways_hash;
  55
  56 static int attr_debug_level=1;
  57 static int nodeid,wayid;
  58 static int report,phase;
  59 static int ignore_unkown = 0, coverage=0;
  60
  61 static char *attrmap={
  62         "n      *=*                     point_unkn\n"
  63         "n      amenity=hospital        poi_hospital\n"
  64         "n      amenity=atm             poi_bank\n"
  65         "n      amenity=bank            poi_bank\n"
  66         "n      amenity=pub             poi_bar\n"
  67         "n      amenity=cafe            poi_cafe\n"
  68         "n      amenity=bus_station     poi_bus_station\n"
  69         "n      amenity=parking         poi_car_parking\n"
  70         "n      amenity=cinema          poi_cinema\n"
  71         "n      amenity=fire_station    poi_firebrigade\n"
  72         "n      amenity=fuel            poi_fuel\n"
  73         "n      amenity=courthouse      poi_justice\n"
  74         "n      amenity=library         poi_library\n"
  75         "n      amenity=pharmacy        poi_pharmacy\n"
  76         "n      amenity=place_of_worship,religion=christian     poi_church\n"
  77         "n      amenity=police          poi_police\n"
  78         "n      amenity=post_office     poi_post\n"
  79         "n      amenity=post_box        poi_post\n"
  80         "n      amenity=public_building poi_public_office\n"
  81         "n      amenity=restaurant      poi_restaurant\n"
  82         "n      amenity=fast_food       poi_fastfood\n"
  83         "n      amenity=toilets         poi_restroom\n"
  84         "n      amenity=school          poi_school\n"
  85         "n      amenity=university      poi_school\n"
  86         "n      amenity=college         poi_school\n"
  87         "n      amenity=telephone       poi_telephone\n"
  88         "n      amenity=theatre         poi_theater\n"
  89         "n      amenity=townhall                poi_townhall\n"
  90         "n      highway=bus_stop        poi_bus_stop\n"
  91         "n      highway=mini_roundabout mini_roundabout\n"
  92         "n      highway=motorway_junction       highway_exit\n"
  93         "n      highway=traffic_signals traffic_signals\n"
  94         "n      highway=turning_circle  turning_circle\n"
  95         "n      leisure=slipway         poi_boat_ramp\n"
  96         "n      leisure=fishing         poi_fish\n"
  97         "n      sport=golf      poi_golf\n"
  98         "n      leisure=golf_course     poi_golf\n"
  99         "n      leisure=marina          poi_marine\n"
 100         "n      leisure=sports_centre   poi_sport\n"
 101         "n      leisure=stadium         poi_stadium\n"
 102         "n      shop=supermarket        poi_shopping\n"
 103         "n      shop=convenience        poi_shop_grocery\n"
 104         "n      tourism=attraction      poi_attraction\n"
 105         "n      tourism=camp_site       poi_camp_rv\n"
 106         "n      tourism=caravan_site    poi_camp_rv\n"
 107         "n      tourism=hotel           poi_hotel\n"
 108         "n      tourism=motel           poi_hotel\n"
 109         "n      tourism=guest_house     poi_hotel\n"
 110         "n      tourism=hostel          poi_hotel\n"
 111         "n      tourism=information     poi_information\n"
 112         "n      tourism=museum          poi_museum_history\n"
 113         "n      tourism=picnic_site     poi_picnic\n"
 114         "n      tourism=theme_park      poi_resort\n"
 115         "n      tourism=zoo             poi_zoo\n"
 116         "n      amenity=grave_yard      poi_cemetery\n"
 117         "n      landuse=cemetery        poi_cemetery\n"
 118         "n      military=airfield       poi_military\n"
 119         "n      military=bunker         poi_military\n"
 120         "n      military=barracks       poi_military\n"
 121         "n      military=range          poi_military\n"
 122         "n      military=danger_area    poi_danger_area\n"
 123         "n      sport=swimming  poi_swimming\n"
 124         "n      sport=skiing            poi_skiing\n"
 125         "n      aeroway=aerodrome       poi_airport\n"
 126         "n      aeroway=airport         poi_airport\n"
 127         "n      aeroway=terminal        poi_airport\n"
 128         "n      aeroway=helipad         poi_heliport\n"
 129         "n      man_made=tower  poi_tower\n"
 130         "n      natural=bay             poi_bay\n"
 131         "n      natural=peak            poi_peak\n"
 132         "n      place=suburb            district_label\n"
 133         "n      place=city              town_label_2e5\n"
 134         "n      place=town              town_label_2e4\n"
 135         "n      place=village           town_label_2e3\n"
 136         "n      place=hamlet            town_label_2e2\n"
 137         "n      place=locality          town_label_2e0\n"
 138         "n      railway=halt            poi_rail_halt\n"
 139         "n      railway=level_crossing          poi_level_crossing\n"
 140         "n      railway=station         poi_rail_station\n"
 141         "n      railway=tram_stop               poi_rail_tram_stop\n"
 142         "n      amenity=park_bench              poi_bench\n"
 143         "n      amenity=bench           poi_bench\n"
 144         "n      amenity=biergarten              poi_biergarten\n"
 145         "n      historic=boundary_stone         poi_boundary_stone\n"
 146         "n      historic=castle         poi_castle\n"
 147         "n      historic=ruins          poi_ruins\n"
 148         "n      Annehmlichkeit=Hochsitz         poi_hunting_stand\n"
 149         "n      amenity=hunting_stand           poi_hunting_stand\n"
 150         "n      historic=memorial               poi_memorial\n"
 151         "n      historic=monument               poi_monument\n"
 152         "n      amenity=shelter         poi_shelter\n"
 153         "n      amenity=fountain                poi_fountain\n"
 154         "n      amenity=drinking_water          poi_potable_water\n"
 155         "n      amenity=toilets         poi_toilets\n"
 156         "n      tourism=viewpoint               poi_viewpoint\n"
 157         "w      *=*                     street_unkn\n"
 158         "w      contour_ext=elevation_major     height_line_1\n"
 159         "w      contour_ext=elevation_medium    height_line_2\n"
 160         "w      contour_ext=elevation_minor     height_line_3\n"
 161         "w      amenity=place_of_worship        poly_building\n"
 162         "w      building=*      poly_building\n"
 163         "w      aeroway=aerodrome       poly_airport\n"
 164         "w      aeroway=apron           poly_apron\n"
 165         "w      aeroway=runway          aeroway_runway\n"
 166         "w      aeroway=taxiway         aeroway_taxiway\n"
 167         "w      aeroway=terminal        poly_terminal\n"
 168         "w      highway=bridleway       bridleway\n"
 169         "w      highway=path,horse=designated   bridleway\n"
 170         "w      highway=cycleway        cycleway\n"
 171         "w      highway=cyclepath       cycleway\n"
 172         "w      highway=path,bicycle=designated cycleway\n"
 173         "w      highway=footway         footway\n"
 174         "w      piste:type=nordic       piste_nordic\n"
 175         "w      highway=footway,piste:type=nordic       footway_and_piste_nordic\n"
 176         "w      route=ski       piste_nordic\n"
 177         "w      piste:type=downhill,piste:difficulty=novice     piste_downhill_novice\n"
 178         "w      piste:type=downhill,piste:difficulty=easy       piste_downhill_easy\n"
 179         "w      piste:type=downhill,piste:difficulty=intermediate       piste_downhill_intermediate\n"
 180         "w      piste:type=downhill,piste:difficulty=advanced   piste_downhill_advanced\n"
 181         "w      piste:type=downhill,piste:difficulty=expert     piste_downhill_expert\n"
 182         "w      piste:type=downhill,piste:difficulty=freeride   piste_downhill_freeride\n"
 183         "w      highway=path,foot=designated    footway\n"
 184         "w      highway=steps   steps\n"
 185         "w      highway=path    path\n"
 186         "w      highway=path,sac_scale=hiking   hiking\n"
 187         "w      highway=path,sac_scale=mountain_hiking  hiking_mountain\n"
 188         "w      highway=path,sac_scale=demanding_mountain_hiking        hiking_mountain_demanding\n"
 189         "w      highway=path,sac_scale=alpine_hiking    hiking_alpine\n"
 190         "w      highway=path,sac_scale=demanding_alpine_hiking  hiking_alpine_demanding\n"
 191         "w      highway=path,sac_scale=difficult_alpine_hiking  hiking_alpine_difficult\n"
 192         "w      highway=unsurfaced      track_gravelled\n"
 193         "w      highway=track           track_gravelled\n"
 194         "w      highway=track,tracktype=grade1          track_paved\n"
 195         "w      highway=track,surface=paved             track_paved\n"
 196         "w      highway=track,tracktype=grade2          track_gravelled\n"
 197         "w      highway=track,surface=gravel            track_gravelled\n"
 198         "w      highway=track,tracktype=grade3          track_unpaved\n"
 199         "w      highway=track,surface=unpaved           track_unpaved\n"
 200         "w      highway=track,tracktype=grade4          track_ground\n"
 201         "w      highway=track,surface=ground            track_ground\n"
 202         "w      highway=track,tracktype=grade5          track_grass\n"
 203         "w      highway=track,surface=grass             track_grass\n"
 204         "w      highway=parking_lane    street_parking_lane\n"
 205         "w      highway=service,service=parking_aisle   street_parking_lane\n"
 206         "w      highway=service         street_service\n"
 207         "w      highway=service,area=1  poly_service\n"
 208         "w      highway=construction            street_construction\n"
 209         "w      highway=pedestrian      street_pedestrian\n"
 210         "w      highway=pedestrian,area=1       poly_pedestrian\n"
 211         "w      highway=plaza   poly_plaza\n"
 212         "w      landuse=plaza   poly_plaza\n"
 213         "w      highway=residential     street_1_city\n"
 214         "w      highway=residential,area=1      poly_street_1\n"
 215         "w      highway=living_street           living_street\n"
 216         "w      highway=unclassified    street_1_city\n"
 217         "w      highway=unclassified,area=1     poly_street_1\n"
 218         "w      highway=road    street_1_city\n"
 219         "w      highway=minor           street_1_land\n"
 220         "w      highway=tertiary        street_2_city\n"
 221         "w      highway=tertiary,area=1 poly_street_2\n"
 222         "w      highway=tertiary_link   ramp\n"
 223         "w      highway=secondary       street_3_city\n"
 224         "w      highway=secondary,area=1        poly_street_3\n"
 225         "w      highway=secondary_link  ramp\n"
 226         "w      highway=primary         street_4_city\n"
 227         "w      highway=primary_link    ramp\n"
 228         "w      highway=trunk           street_4_city\n"
 229         "w      highway=trunk_link      ramp\n"
 230         "w      highway=motorway        highway_city\n"
 231         "w      highway=motorway_link   ramp\n"
 232         "w      historic=town gate      poly_building\n"
 233         "w      cycleway=track          cycleway\n"
 234         "w      amenity=parking         poly_car_parking\n"
 235         "w      landuse=allotments      poly_allotments\n"
 236         "w      landuse=cemetery        poly_cemetery\n"
 237         "w      amenity=grave_yard      poly_cemetery\n"
 238         "w      landuse=forest          poly_wood\n"
 239         "w      landuse=industrial      poly_industry\n"
 240         "w      landuse=residential     poly_town\n"
 241         "w      landuse=residential,area=1      poly_town\n"
 242         "w      landuse=farmyard        poly_town\n"
 243         "w      amenity=college poly_college\n"
 244         "w      amenity=university      poly_university\n"
 245         "w      tourism=camp_site       poly_camp_site\n"
 246         "w      tourism=caravan_site    poly_caravan_site\n"
 247         "w      tourism=picnic_site     poly_picnic_site\n"
 248         "w      tourism=theme_park      poly_theme_park\n"
 249         "w      tourism=attraction      poly_attraction\n"
 250         "w      tourism=zoo     poly_zoo\n"
 251         "w      tourism=artwork poly_artwork\n"
 252         "w      historic=archaeological_site    poly_archaeological_site\n"
 253         "w      historic=ruins  poly_ruins\n"
 254         "w      historic=battlefield    poly_battlefield\n"
 255         "w      landuse=quarry  poly_quarry\n"
 256         "w      landuse=landfill        poly_landfill\n"
 257         "w      landuse=retail  poly_retail\n"
 258         "w      landuse=commercial      poly_commercial\n"
 259         "w      landuse=brownfield      poly_brownfield\n"
 260         "w      landuse=greenfield      poly_greenfield\n"
 261         "w      landuse=construction    poly_construction\n"
 262         "w      landuse=railway poly_railway\n"
 263         "w      natural=glacier poly_glacier\n"
 264         "w      natural=scree   poly_scree\n"
 265         "w      natural=scrub   poly_scrub\n"
 266         "w      natural=fell    poly_fell\n"
 267         "w      natural=heath   poly_heath\n"
 268         "w      natural=marsh   poly_marsh\n"
 269         "w      natural=mud     poly_mud\n"
 270         "w      natural=beach   poly_beach\n"
 271         "w      natural=land    poly_land\n"
 272         "w      landuse=basin   poly_basin\n"
 273         "w      landuse=reservoir       poly_reservoir\n"
 274         "w      landuse=farm    poly_farm\n"
 275         "w      landuse=farmland        poly_farm\n"
 276         "w      landuse=vineyard        poly_farm\n"
 277         "w      leisure=park            poly_park\n"
 278         "w      landuse=village_green           poly_village_green\n"
 279         "w      landuse=recreation_ground               poly_recreation_ground\n"
 280         "w      natural=wood            poly_wood\n"
 281         "w      natural=water           poly_water\n"
 282         "w      natural=coastline       water_line\n"
 283         "w      place=suburb            poly_town\n"
 284         "w      place=town              poly_town\n"
 285         "w      landuse=military        poly_military\n"
 286         "w      military=airfield       poly_airfield\n"
 287         "w      military=barracks       poly_barracks\n"
 288         "w      military=danger_area    poly_danger_area\n"
 289         "w      military=range  poly_range\n"
 290         "w      military=naval_base     poly_naval_base\n"
 291         "w      power=line      powerline\n"
 292         "w      railway=rail            rail\n"
 293         "w      railway=narrow_gauge    rail_narrow_gauge\n"
 294         "w      railway=light_rail              rail_light\n"
 295         "w      railway=subway          rail_subway\n"
 296         "w      railway=tram            rail_tram\n"
 297         "w      railway=monorail                rail_mono\n"
 298         "w      railway=preserved               rail_preserved\n"
 299         "w      railway=disused         rail_disused\n"
 300         "w      railway=abandoned               rail_abandoned\n"
 301         "w      aerialway=cable_car             lift_cable_car\n"
 302         "w      aerialway=chair_lift            lift_chair\n"
 303         "w      aerialway=drag_lift             lift_drag\n"
 304         "w      leisure=golf_course     poly_golf_course\n"
 305         "w      sport=*         poly_sport\n"
 306         "w      leisure=sports_centre           poly_sport\n"
 307         "w      leisure=stadium         poly_sports_stadium\n"
 308         "w      leisure=track           poly_sports_track\n"
 309         "w      leisure=pitch           poly_sports_pitch\n"
 310         "w      leisure=water_park              poly_water_park\n"
 311         "w      leisure=marina          poly_marina\n"
 312         "w      leisure=fishing         poly_fishing\n"
 313         "w      leisure=garden          poly_garden\n"
 314         "w      leisure=common          poly_common\n"
 315         "w      leisure=playground              poly_playground\n"
 316         "w      leisure=nature_reserve          poly_nature_reserve\n"
 317         "w      waterway=canal          water_canal\n"
 318         "w      waterway=river          water_river\n"
 319         "w      waterway=stream         water_stream\n"
 320         "w      waterway=drain          water_drain\n"
 321         "w      waterway=riverbank      poly_water\n"
 322         "w      boundary=administrative border_country\n"
 323         "w      boundary=civil  border_civil\n"
 324         "w      boundary=political      border_political\n"
 325         "w      boundary=national_park  border_national_park\n"
 326         "w      route=ferry             ferry\n"
 327         "w      highway=bus_guideway            bus_guideway\n"
 328 };
 329
 330 struct coord {
 331         int x;
 332         int y;
 333 } coord_buffer[65536];
 334
 335 #define IS_REF(c) ((c).x >= (1 << 30))
 336 #define REF(c) ((c).y)
 337 #define SET_REF(c,ref) do { (c).x = 1 << 30; (c).y = ref ; } while(0)
 338
 339 struct rect {
 340         struct coord l,h;
 341 };
 342
 343 static void bbox_extend(struct coord *c, struct rect *r);
 344
 345 #ifdef GENERATE_INDEX
 346
 347 static GHashTable *aux_tile_hash;
 348 GList *aux_tile_list;
 349
 350 struct country_table {
 351         int countryid;
 352         char *names;
 353         FILE *file;
 354         int size;
 355         int count;
 356         struct rect r;
 357 } country_table[] = {
 358         { 40,"Austria,�sterreich,AUT"},
 359         {124,"Canada"},
 360         {208,"Denmark,Danmark,DK"},
 361         {246,"Finland,Suomi"},
 362         {250,"France,R�publique fran�aise,FR"},
 363         {276,"Germany,Deutschland,Bundesrepublik Deutschland"},
 364         {380,"Italy,Italia"},
 365         {578,"Norway,Norge,Noreg,NO"},
 366         {752,"Sweden,Sverige,Konungariket Sverige,SE"},
 367         {528,"Nederland,The Netherlands,Niederlande,NL"},
 368         {724,"Spain,Espana,Espana,Reino de Espana"},
 369         {756,"Schweiz"},
 370         {840,"USA"}
 371 };
 372
 373 static GHashTable *country_table_hash;
 374 #endif
 375
 376 struct attr_mapping {
 377         enum item_type type;
 378         int attr_present_idx_count;
 379         int attr_present_idx[0];
 380 };
 381
 382 static struct attr_mapping **attr_mapping_node;
 383 static int attr_mapping_node_count;
 384 static struct attr_mapping **attr_mapping_way;
 385 static int attr_mapping_way_count;
 386
 387 static char *attr_present;
 388 static int attr_present_count;
 389 static GHashTable *attr_hash;
 390
 391
 392 static GHashTable *strings_hash = NULL;
 393
 394
 395 static char* string_hash_lookup( const char* key )
 396 {
 397         char* key_ptr = NULL;
 398
 399         if ( strings_hash == NULL ) {
 400                 strings_hash = g_hash_table_new(g_str_hash, g_str_equal);
 401         }
 402
 403         if ( ( key_ptr = g_hash_table_lookup(strings_hash, key )) == NULL ) {
 404                 key_ptr = g_strdup( key );
 405                 g_hash_table_insert(strings_hash, key_ptr,  (gpointer)key_ptr );
 406
 407         }
 408         return key_ptr;
 409 }
 410
 411 static void
 412 build_attrmap_line(char *line)
 413 {
 414         char *t=NULL,*kvl=NULL,*i=NULL,*p,*kv;
 415         struct attr_mapping ***attr_mapping_curr,*attr_mapping=g_malloc0(sizeof(struct attr_mapping));
 416         int idx,attr_mapping_count=0,*attr_mapping_curr_count;
 417         t=line;
 418         p=strchr(t,'\t');
 419         if (p) {
 420                 while (*p == '\t')
 421                         *p++='\0';
 422                 kvl=p;
 423                 p=strchr(kvl,'\t');
 424         }
 425         if (p) {
 426                 while (*p == '\t')
 427                         *p++='\0';
 428                 i=p;
 429         }
 430         if (t[0] == 'w') {
 431                 if (! i)
 432                         i="street_unkn";
 433                 attr_mapping_curr=&attr_mapping_way;
 434                 attr_mapping_curr_count=&attr_mapping_way_count;
 435         } else {
 436                 if (! i)
 437                         i="point_unkn";
 438                 attr_mapping_curr=&attr_mapping_node;
 439                 attr_mapping_curr_count=&attr_mapping_node_count;
 440         }
 441         attr_mapping->type=item_from_name(i);
 442         while ((kv=strtok(kvl, ","))) {
 443                 kvl=NULL;
 444                 if (!(idx=(int)g_hash_table_lookup(attr_hash, kv))) {
 445                         idx=attr_present_count++;
 446                         g_hash_table_insert(attr_hash, kv, (gpointer) idx);
 447                 }
 448                 attr_mapping=g_realloc(attr_mapping, sizeof(struct attr_mapping)+(attr_mapping_count+1)*sizeof(int));
 449                 attr_mapping->attr_present_idx[attr_mapping_count++]=idx;
 450                 attr_mapping->attr_present_idx_count=attr_mapping_count;
 451         }
 452         *attr_mapping_curr=g_realloc(*attr_mapping_curr, sizeof(**attr_mapping_curr)*(*attr_mapping_curr_count+1));
 453         (*attr_mapping_curr)[(*attr_mapping_curr_count)++]=attr_mapping;
 454 }
 455
 456 static void
 457 build_attrmap(char *map)
 458 {
 459         char *p;
 460         attr_hash=g_hash_table_new(g_str_hash, g_str_equal);
 461         attr_present_count=1;
 462         while (map) {
 463                 p=strchr(map,'\n');
 464                 if (p)
 465                         *p++='\0';
 466                 if (strlen(map))
 467                         build_attrmap_line(map);
 468                 map=p;
 469         }
 470         attr_present=g_malloc0(sizeof(*attr_present)*attr_present_count);
 471 }
 472
 473 #ifdef GENERATE_INDEX
 474 static void
 475 build_countrytable(void)
 476 {
 477         int i;
 478         char *names,*str,*tok;
 479         country_table_hash=g_hash_table_new(g_str_hash, g_str_equal);
 480         for (i = 0 ; i < sizeof(country_table)/sizeof(struct country_table) ; i++) {
 481                 names=g_strdup(country_table[i].names);
 482                 str=names;
 483                 while ((tok=strtok(str, ","))) {
 484                         str=NULL;
 485                         g_hash_table_insert(country_table_hash, tok,  (gpointer)&country_table[i]);
 486                 }
 487         }
 488 }
 489 #endif
 490
 491
 492
 493 static int processed_nodes, processed_nodes_out, processed_ways, processed_relations, processed_tiles;
 494 static int in_way, in_node, in_relation;
 495
 496 static void
 497 sig_alrm(int sig)
 498 {
 499 #ifndef _WIN32
 500         signal(SIGALRM, sig_alrm);
 501 #endif
 502         alarm(30);
 503         fprintf(stderr,"PROGRESS%d: Processed %d nodes (%d out) %d ways %d relations %d tiles\n", phase, processed_nodes, processed_nodes_out, processed_ways, processed_relations, processed_tiles);
 504 }
 505
 506 struct item_bin {
 507         int len;
 508         enum item_type type;
 509         int clen;
 510 } item;
 511
 512 struct attr_bin {
 513         int len;
 514         enum attr_type type;
 515 };
 516
 517 struct attr_bin label_attr = {
 518         0, attr_label
 519 };
 520 char label_attr_buffer[BUFFER_SIZE];
 521
 522 struct attr_bin town_name_attr = {
 523         0, attr_town_name
 524 };
 525
 526 struct attr_bin street_name_attr = {
 527         0, attr_street_name
 528 };
 529
 530 struct attr_bin street_name_systematic_attr = {
 531         0, attr_street_name_systematic
 532 };
 533 char street_name_systematic_attr_buffer[BUFFER_SIZE];
 534
 535 struct attr_bin debug_attr = {
 536         0, attr_debug
 537 };
 538 char debug_attr_buffer[BUFFER_SIZE];
 539
 540 struct attr_bin flags_attr = {
 541         0, attr_flags
 542 };
 543 int flags_attr_value;
 544
 545 char is_in_buffer[BUFFER_SIZE];
 546
 547
 548 static void write_zipmember(FILE *out, FILE *dir_out, char *name, int filelen, char *data, int data_size, int compression_level);
 549
 550 static void
 551 pad_text_attr(struct attr_bin *a, char *buffer)
 552 {
 553         int l;
 554         if (buffer && buffer[0]) {
 555                 l=strlen(buffer)+1;
 556                 while (l % 4)
 557                         buffer[l++]='\0';
 558                 a->len=l/4+1;
 559         } else
 560                 a->len=0;
 561 }
 562
 563 static int
 564 xml_get_attribute(char *xml, char *attribute, char *buffer, int buffer_size)
 565 {
 566         int len=strlen(attribute);
 567         char *pos,*i,s,attr[len+2];
 568         strcpy(attr, attribute);
 569         strcpy(attr+len, "=");
 570         pos=strstr(xml, attr);
 571         if (! pos)
 572                 return 0;
 573         pos+=len+1;
 574         s=*pos++;
 575         if (! s)
 576                 return 0;
 577         i=strchr(pos, s);
 578         if (! i)
 579                 return 0;
 580         if (i - pos > buffer_size) {
 581                 fprintf(stderr,"Buffer overflow %d vs %d\n", i-pos, buffer_size);
 582                 return 0;
 583         }
 584         strncpy(buffer, pos, i-pos);
 585         buffer[i-pos]='\0';
 586         return 1;
 587 }
 588
 589 static int node_is_tagged;
 590
 591 static void
 592 add_tag(char *k, char *v)
 593 {
 594         GHashTable *value_hash;
 595         enum item_type type;
 596         int idx,level=2;
 597         char buffer[BUFFER_SIZE*2+2];
 598         if (! strcmp(k,"ele"))
 599                 level=9;
 600         if (! strcmp(k,"time"))
 601                 level=9;
 602         if (! strcmp(k,"created_by"))
 603                 level=9;
 604         if (! strncmp(k,"tiger:",6) || !strcmp(k,"AND_nodes"))
 605                 level=9;
 606         if (! strcmp(k,"converted_by") || ! strcmp(k,"source"))
 607                 level=8;
 608         if (! strncmp(k,"osmarender:",11) || !strncmp(k,"svg:",4))
 609                 level=8;
 610         if (! strcmp(k,"layer"))
 611                 level=7;
 612         if (! strcasecmp(v,"true") || ! strcasecmp(v,"yes"))
 613                 v="1";
 614         if (! strcmp(k,"oneway")) {
 615                 if (!strcmp(v,"1")) {
 616                         flags_attr_value=AF_ONEWAY;
 617                         flags_attr.len=2;
 618                 }
 619                 if (! strcmp(v,"-1")) {
 620                         flags_attr_value=AF_ONEWAYREV;
 621                         flags_attr.len=2;
 622                 }
 623                 if (!in_way)
 624                         level=6;
 625                 else
 626                         level=5;
 627         }
 628         if (! strcmp(k,"junction")) {
 629                 if (! strcmp(v,"roundabout")) {
 630                         flags_attr_value=AF_ONEWAY;
 631                         flags_attr.len=2;
 632                 }
 633         }
 634         if (! strcmp(k,"maxspeed")) {
 635                 level=5;
 636         }
 637         if (! strcmp(k,"bicycle")) {
 638                 level=5;
 639         }
 640         if (! strcmp(k,"foot")) {
 641                 level=5;
 642         }
 643         if (! strcmp(k,"note"))
 644                 level=5;
 645         if (! strcmp(k,"name")) {
 646                 strcpy(label_attr_buffer, v);
 647                 pad_text_attr(&label_attr, label_attr_buffer);
 648                 level=5;
 649         }
 650         if (! strcmp(k,"ref")) {
 651                 if (in_way) {
 652                         strcpy(street_name_systematic_attr_buffer, v);
 653                         pad_text_attr(&street_name_systematic_attr, street_name_systematic_attr_buffer);
 654                 }
 655                 level=5;
 656         }
 657         if (! strcmp(k,"is_in")) {
 658                 strcpy(is_in_buffer, v);
 659                 level=5;
 660         }
 661         if (! strcmp(k,"gnis:ST_alpha")) {
 662                 /*      assume a gnis tag means it is part of the USA:
 663                         http://en.wikipedia.org/wiki/Geographic_Names_Information_System
 664                         many US towns do not have is_in tags
 665                 */
 666                 strcpy(is_in_buffer, "USA");
 667                 level=5;
 668         }
 669         if (! strcmp(k,"lanes")) {
 670                 level=5;
 671         }
 672         if (attr_debug_level >= level) {
 673                 int bytes_left = sizeof( debug_attr_buffer ) - strlen(debug_attr_buffer) - 1;
 674                 if ( bytes_left > 0 )
 675                 {
 676                         snprintf(debug_attr_buffer+strlen(debug_attr_buffer), bytes_left,  " %s=%s", k, v);
 677                         debug_attr_buffer[ sizeof( debug_attr_buffer ) -  1 ] = '\0';
 678                         node_is_tagged=1;
 679                 }
 680         }
 681         if (level < 6)
 682                 node_is_tagged=1;
 683         if (level >= 5)
 684                 return;
 685
 686         strcpy(buffer,"*=*");
 687         if ((idx=(int)g_hash_table_lookup(attr_hash, buffer)))
 688                 attr_present[idx]=1;
 689
 690         sprintf(buffer,"%s=*", k);
 691         if ((idx=(int)g_hash_table_lookup(attr_hash, buffer)))
 692                 attr_present[idx]=2;
 693
 694         sprintf(buffer,"*=%s", v);
 695         if ((idx=(int)g_hash_table_lookup(attr_hash, buffer)))
 696                 attr_present[idx]=2;
 697
 698         sprintf(buffer,"%s=%s", k, v);
 699         if ((idx=(int)g_hash_table_lookup(attr_hash, buffer)))
 700                 attr_present[idx]=4;
 701 }
 702
 703 struct entity {
 704         char *entity;
 705         char c;
 706 } entities[]= {
 707         {"&quot;",'"'},
 708         {"&apos;",'\''},
 709         {"&amp;",'&'},
 710         {"&lt;",'<'},
 711         {"&gt;",'>'},
 712 };
 713
 714 static void
 715 decode_entities(char *buffer)
 716 {
 717         char *pos=buffer;
 718         int i,len,found;
 719
 720         while ((pos=strchr(pos, '&'))) {
 721                 found=0;
 722                 for (i = 0 ; i < sizeof(entities)/sizeof(struct entity); i++) {
 723                         len=strlen(entities[i].entity);
 724                         if (!strncmp(pos, entities[i].entity, len)) {
 725                                 *pos=entities[i].c;
 726                                 memmove(pos+1, pos+len, strlen(pos+len)+1);
 727                                 found=1;
 728                                 break;
 729                         }
 730                 }
 731                 pos++;
 732         }
 733 }
 734
 735 static int
 736 parse_tag(char *p)
 737 {
 738         char k_buffer[BUFFER_SIZE];
 739         char v_buffer[BUFFER_SIZE];
 740         if (!xml_get_attribute(p, "k", k_buffer, BUFFER_SIZE))
 741                 return 0;
 742         if (!xml_get_attribute(p, "v", v_buffer, BUFFER_SIZE))
 743                 return 0;
 744         decode_entities(v_buffer);
 745         add_tag(k_buffer, v_buffer);
 746         return 1;
 747 }
 748
 749
 750 struct buffer {
 751         int malloced_step;
 752         size_t malloced;
 753         unsigned char *base;
 754         size_t size;
 755 };
 756
 757 static struct tile_head {
 758         int num_subtiles;
 759         int total_size;
 760         char *name;
 761         char *zip_data;
 762         int total_size_used;
 763         int zipnum;
 764         int process;
 765         struct tile_head *next;
 766         // char subtiles[0];
 767 } *tile_head_root;
 768
 769
 770 int coord_count;
 771
 772 struct node_item {
 773         int id;
 774         char ref_node;
 775         char ref_way;
 776         char ref_ref;
 777         char dummy;
 778         struct coord c;
 779 };
 780
 781 static struct buffer node_buffer = {
 782         64*1024*1024,
 783 };
 784
 785
 786 static char** th_get_subtile( const struct tile_head* th, int idx )
 787 {
 788         char* subtile_ptr = NULL;
 789         subtile_ptr = (char*)th + sizeof( struct tile_head ) + idx * sizeof( char *);
 790         return (char**)subtile_ptr;
 791 }
 792
 793 static void
 794 extend_buffer(struct buffer *b)
 795 {
 796         b->malloced+=b->malloced_step;
 797         b->base=realloc(b->base, b->malloced);
 798         if (b->base == NULL) {
 799                 fprintf(stderr,"realloc of %d bytes failed\n",b->malloced);
 800                 exit(1);
 801         }
 802
 803 }
 804
 805 int nodeid_last;
 806 GHashTable *node_hash;
 807
 808 static void
 809 node_buffer_to_hash(void)
 810 {
 811         int i,count=node_buffer.size/sizeof(struct node_item);
 812         struct node_item *ni=(struct node_item *)node_buffer.base;
 813         for (i = 0 ; i < count ; i++)
 814                 g_hash_table_insert(node_hash, (gpointer)(ni[i].id), (gpointer)i);
 815 }
 816
 817 static struct node_item *ni;
 818
 819 static void
 820 add_node(int id, double lat, double lon)
 821 {
 822         if (node_buffer.size + sizeof(struct node_item) > node_buffer.malloced)
 823                 extend_buffer(&node_buffer);
 824         node_is_tagged=0;
 825         nodeid=id;
 826         item.type=type_point_unkn;
 827         label_attr.len=0;
 828         town_name_attr.len=0;
 829         debug_attr.len=0;
 830         is_in_buffer[0]='\0';
 831         sprintf(debug_attr_buffer,"nodeid=%d", nodeid);
 832         ni=(struct node_item *)(node_buffer.base+node_buffer.size);
 833         ni->id=id;
 834         ni->ref_node=0;
 835         ni->ref_way=0;
 836         ni->ref_ref=0;
 837         ni->dummy=0;
 838         ni->c.x=lon*6371000.0*M_PI/180;
 839         ni->c.y=log(tan(M_PI_4+lat*M_PI/360))*6371000.0;
 840         node_buffer.size+=sizeof(struct node_item);
 841         if (! node_hash) {
 842                 if (ni->id > nodeid_last) {
 843                         nodeid_last=ni->id;
 844                 } else {
 845                         fprintf(stderr,"INFO: Nodes out of sequence (new %d vs old %d), adding hash\n", ni->id, nodeid_last);
 846                         node_hash=g_hash_table_new(NULL, NULL);
 847                         node_buffer_to_hash();
 848                 }
 849         } else
 850                 if (!g_hash_table_lookup(node_hash, (gpointer)(ni->id)))
 851                         g_hash_table_insert(node_hash, (gpointer)(ni->id), (gpointer)(ni-(struct node_item *)node_buffer.base));
 852                 else {
 853                         node_buffer.size-=sizeof(struct node_item);
 854                         nodeid=0;
 855                 }
 856
 857 }
 858
 859 static int
 860 parse_node(char *p)
 861 {
 862         char id_buffer[BUFFER_SIZE];
 863         char lat_buffer[BUFFER_SIZE];
 864         char lon_buffer[BUFFER_SIZE];
 865         if (!xml_get_attribute(p, "id", id_buffer, BUFFER_SIZE))
 866                 return 0;
 867         if (!xml_get_attribute(p, "lat", lat_buffer, BUFFER_SIZE))
 868                 return 0;
 869         if (!xml_get_attribute(p, "lon", lon_buffer, BUFFER_SIZE))
 870                 return 0;
 871         add_node(atoi(id_buffer), atof(lat_buffer), atof(lon_buffer));
 872         return 1;
 873 }
 874
 875
 876 static struct node_item *
 877 node_item_get(int id)
 878 {
 879         struct node_item *ni=(struct node_item *)(node_buffer.base);
 880         int count=node_buffer.size/sizeof(struct node_item);
 881         int interval=count/4;
 882         int p=count/2;
 883         if (node_hash) {
 884                 int i;
 885                 i=(int)(g_hash_table_lookup(node_hash, (gpointer)id));
 886                 return ni+i;
 887         }
 888         while (ni[p].id != id) {
 889 #if 0
 890                 fprintf(stderr,"p=%d count=%d interval=%d id=%d ni[p].id=%d\n", p, count, interval, id, ni[p].id);
 891 #endif
 892                 if (ni[p].id < id) {
 893                         p+=interval;
 894                         if (interval == 1) {
 895                                 if (p >= count)
 896                                         return NULL;
 897                                 if (ni[p].id > id)
 898                                         return NULL;
 899                         } else {
 900                                 if (p >= count)
 901                                         p=count-1;
 902                         }
 903                 } else {
 904                         p-=interval;
 905                         if (interval == 1) {
 906                                 if (p < 0)
 907                                         return NULL;
 908                                 if (ni[p].id < id)
 909                                         return NULL;
 910                         } else {
 911                                 if (p < 0)
 912                                         p=0;
 913                         }
 914                 }
 915                 if (interval > 1)
 916                         interval/=2;
 917         }
 918
 919         return &ni[p];
 920 }
 921
 922 static void
 923 node_ref_way(int id)
 924 {
 925         struct node_item *ni=node_item_get(id);
 926         if (! ni) {
 927                 fprintf(stderr,"WARNING: node id %d not found\n", id);
 928                 return;
 929         }
 930         ni->ref_way++;
 931 }
 932
 933
 934 static void
 935 add_way(int id)
 936 {
 937         wayid=id;
 938         coord_count=0;
 939         item.type=type_street_unkn;
 940         label_attr.len=0;
 941         street_name_attr.len=0;
 942         street_name_systematic_attr.len=0;
 943         debug_attr.len=0;
 944         flags_attr.len=0;
 945         sprintf(debug_attr_buffer,"wayid=%d", wayid);
 946 }
 947
 948 static int
 949 parse_way(char *p)
 950 {
 951         char id_buffer[BUFFER_SIZE];
 952         if (!xml_get_attribute(p, "id", id_buffer, BUFFER_SIZE))
 953                 return 0;
 954         add_way(atoi(id_buffer));
 955         return 1;
 956 }
 957
 958 static int
 959 parse_relation(char *p)
 960 {
 961         debug_attr_buffer[0]='\0';
 962         return 1;
 963 }
 964
 965 static void
 966 item_buffer_set_type(char *buffer, enum item_type type)
 967 {
 968         struct item_bin *ib=(struct item_bin *) buffer;
 969         ib->clen=0;
 970         ib->len=2;
 971         ib->type=type;
 972 }
 973
 974 static void
 975 item_buffer_add_coord(char *buffer, struct coord *c, int count)
 976 {
 977         struct item_bin *ib=(struct item_bin *) buffer;
 978         struct coord *c2=(struct coord *)(ib+1);
 979         c2+=ib->clen/2;
 980         memcpy(c2, c, count*sizeof(struct coord));
 981         ib->clen+=count*2;
 982         ib->len+=count*2;
 983 }
 984
 985 static void
 986 item_buffer_add_attr(char *buffer, struct attr *attr)
 987 {
 988         struct item_bin *ib=(struct item_bin *) buffer;
 989         struct attr_bin *ab=(struct attr_bin *)((int *)ib+ib->len+1);
 990         if (attr->type >= attr_type_string_begin && attr->type <= attr_type_string_end) {
 991                 ab->type=attr->type;
 992                 strcpy((char *)(ab+1),attr->u.str);
 993                 pad_text_attr(ab, (char *)(ab+1));
 994                 ib->len+=ab->len+1;
 995         }
 996 }
 997
 998 static void
 999 item_buffer_write(char *buffer, FILE *out)
1000 {
1001         struct item_bin *ib=(struct item_bin *) buffer;
1002         fwrite(buffer, (ib->len+1)*4, 1, out);
1003 }
1004
1005 static void
1006 write_attr(FILE *out, struct attr_bin *attr, void *buffer)
1007 {
1008         if (attr->len) {
1009                 fwrite(attr, sizeof(*attr), 1, out);
1010                 fwrite(buffer, (attr->len-1)*4, 1, out);
1011         }
1012 }
1013
1014 static int
1015 attr_longest_match(struct attr_mapping **mapping, int mapping_count, enum item_type *types, int types_count)
1016 {
1017         int i,j,longest=0,ret=0,sum,val;
1018         struct attr_mapping *curr;
1019         for (i = 0 ; i < mapping_count ; i++) {
1020                 sum=0;
1021                 curr=mapping[i];
1022                 for (j = 0 ; j < curr->attr_present_idx_count ; j++) {
1023                         val=attr_present[curr->attr_present_idx[j]];
1024                         if (val)
1025                                 sum+=val;
1026                         else {
1027                                 sum=-1;
1028                                 break;
1029                         }
1030                 }
1031                 if (sum > longest) {
1032                         longest=sum;
1033                         ret=0;
1034                 }
1035                 if (sum > 0 && sum == longest && ret < types_count)
1036                         types[ret++]=curr->type;
1037         }
1038         memset(attr_present, 0, sizeof(*attr_present)*attr_present_count);
1039         return ret;
1040 }
1041
1042 static void
1043 end_way(FILE *out)
1044 {
1045         int alen=0,count;
1046         enum item_type types[5];
1047
1048         if (! out)
1049                 return;
1050         if (dedupe_ways_hash) {
1051                 if (g_hash_table_lookup(dedupe_ways_hash, (gpointer)wayid))
1052                         return;
1053                 g_hash_table_insert(dedupe_ways_hash, (gpointer)wayid, (gpointer)1);
1054         }
1055         count=attr_longest_match(attr_mapping_way, attr_mapping_way_count, types, sizeof(types)/sizeof(enum item_type));
1056         pad_text_attr(&debug_attr, debug_attr_buffer);
1057         if (label_attr.len)
1058                 alen+=label_attr.len+1;
1059         if (street_name_systematic_attr.len)
1060                 alen+=street_name_systematic_attr.len+1;
1061         if (debug_attr.len)
1062                 alen+=debug_attr.len+1;
1063         if (flags_attr.len)
1064                 alen+=flags_attr.len+1;
1065         if (count)
1066                 item.type=types[0];
1067         else
1068                 item.type=type_street_unkn;
1069         if (coverage && item_is_street(item))
1070                 item.type=type_coverage;
1071         item.clen=coord_count*2;
1072         item.len=item.clen+2+alen;
1073         fwrite(&item, sizeof(item), 1, out);
1074         fwrite(coord_buffer, coord_count*sizeof(struct coord), 1, out);
1075         if (item_is_street(item)) {
1076                 street_name_attr.len=label_attr.len;
1077                 write_attr(out, &street_name_attr, label_attr_buffer);
1078         } else
1079                 write_attr(out, &label_attr, label_attr_buffer);
1080         write_attr(out, &street_name_systematic_attr, street_name_systematic_attr_buffer);
1081         write_attr(out, &debug_attr, debug_attr_buffer);
1082         write_attr(out, &flags_attr, &flags_attr_value);
1083 }
1084
1085 static void
1086 end_node(FILE *out)
1087 {
1088         int alen=0,conflict=0,count;
1089         enum item_type types[5];
1090         struct country_table *result=NULL, *lookup;
1091         if (!out || ! node_is_tagged || ! nodeid)
1092                 return;
1093         count=attr_longest_match(attr_mapping_node, attr_mapping_node_count, types, sizeof(types)/sizeof(enum item_type));
1094         pad_text_attr(&debug_attr, debug_attr_buffer);
1095         if (label_attr.len)
1096                 alen+=label_attr.len+1;
1097         if (debug_attr.len)
1098                 alen+=debug_attr.len+1;
1099         if (count)
1100                 item.type=types[0];
1101         else
1102                 item.type=type_point_unkn;
1103         item.clen=2;
1104         item.len=item.clen+2+alen;
1105         fwrite(&item, sizeof(item), 1, out);
1106         fwrite(&ni->c, 1*sizeof(struct coord), 1, out);
1107         if (item_is_town(item)) {
1108                 town_name_attr.len=label_attr.len;
1109                 write_attr(out, &town_name_attr, label_attr_buffer);
1110         } else
1111                 write_attr(out, &label_attr, label_attr_buffer);
1112         write_attr(out, &debug_attr, debug_attr_buffer);
1113 #ifdef GENERATE_INDEX
1114         if (item_is_town(item) && town_name_attr.len) {
1115                 char *tok,*buf=is_in_buffer;
1116                 while ((tok=strtok(buf, ","))) {
1117                         while (*tok==' ')
1118                                 tok++;
1119                         lookup=g_hash_table_lookup(country_table_hash,tok);
1120                         if (lookup) {
1121                                 if (result && result->countryid != lookup->countryid) {
1122                                         fprintf(stderr,"conflict for %s %s country %d vs %d\n", label_attr_buffer, debug_attr_buffer, lookup->countryid, result->countryid);
1123                                         conflict=1;
1124                                 } else
1125                                         result=lookup;
1126                         }
1127                         buf=NULL;
1128                 }
1129                 if (result && !conflict) {
1130                         if (!result->file) {
1131                                 char *name=g_strdup_printf("country_%d.bin.unsorted", result->countryid);
1132                                 result->file=fopen(name,"w");
1133                                 g_free(name);
1134                         }
1135                         if (result->file) {
1136                                 item.clen=2;
1137                                 item.len=item.clen+2+label_attr.len+1;
1138                                 fwrite(&item, sizeof(item), 1, result->file);
1139                                 fwrite(&ni->c, 1*sizeof(struct coord), 1, result->file);
1140                                 write_attr(result->file, &town_name_attr, label_attr_buffer);
1141                                 result->count++;
1142                                 result->size+=(item.clen+3+label_attr.len+1)*4;
1143                         }
1144
1145                 }
1146         }
1147 #endif
1148         processed_nodes_out++;
1149 }
1150
1151 static int
1152 sort_countries_compare(const void *p1, const void *p2)
1153 {
1154         struct item_bin *ib1=*((struct item_bin **)p1),*ib2=*((struct item_bin **)p2);
1155         struct attr_bin *attr1,*attr2;
1156         char *s1,*s2;
1157         assert(ib1->clen==2);
1158         assert(ib2->clen==2);
1159         attr1=(struct attr_bin *)((int *)(ib1+1)+ib1->clen);
1160         attr2=(struct attr_bin *)((int *)(ib2+1)+ib1->clen);
1161         assert(attr1->type == attr_town_name);
1162         assert(attr2->type == attr_town_name);
1163         s1=(char *)(attr1+1);
1164         s2=(char *)(attr2+1);
1165         return strcmp(s1, s2);
1166 #if 0
1167         fprintf(stderr,"sort_countries_compare p1=%p p2=%p %s %s\n",p1,p2,s1,s2);
1168 #endif
1169         return 0;
1170 }
1171
1172 #ifdef GENERATE_INDEX
1173 static void
1174 sort_countries(void)
1175 {
1176         int i,j;
1177         struct country_table *co;
1178         struct coord *c;
1179         struct item_bin *ib;
1180         FILE *f;
1181         char *p,*buffer,**idx,*name;
1182         for (i = 0 ; i < sizeof(country_table)/sizeof(struct country_table) ; i++) {
1183                 co=&country_table[i];
1184                 if (co->file) {
1185                         fclose(co->file);
1186                         co->file=NULL;
1187                 }
1188                 if (co->size) {
1189                         buffer=malloc(co->size);
1190                         assert(buffer != NULL);
1191                         idx=malloc(co->count*sizeof(void *));
1192                         assert(idx != NULL);
1193                         name=g_strdup_printf("country_%d.bin.unsorted", co->countryid);
1194                         f=fopen(name,"r");
1195                         assert(f != NULL);
1196                         fread(buffer, co->size, 1, f);
1197                         fclose(f);
1198                         unlink(name);
1199                         g_free(name);
1200                         p=buffer;
1201                         for (j = 0 ; j < co->count ; j++) {
1202                                 idx[j]=p;
1203                                 p+=(*((int *)p)+1)*4;
1204                         }
1205                         qsort(idx, co->count, sizeof(void *), sort_countries_compare);
1206                         name=g_strdup_printf("country_%d.bin", co->countryid);
1207                         f=fopen(name,"w");
1208                         for (j = 0 ; j < co->count ; j++) {
1209                                 ib=(struct item_bin *)(idx[j]);
1210                                 c=(struct coord *)(ib+1);
1211                                 fwrite(ib, (ib->len+1)*4, 1, f);
1212                                 if (j)
1213                                         bbox_extend(c, &co->r);
1214                                 else
1215                                         co->r.l=*c;
1216                                         co->r.h=*c;
1217                         }
1218                         fclose(f);
1219                 }
1220         }
1221 }
1222 #endif
1223
1224 static void
1225 add_nd(char *p, int ref)
1226 {
1227         int len;
1228         struct node_item *ni;
1229         ni=node_item_get(ref);
1230         if (ni) {
1231 #if 0
1232                 coord_buffer[coord_count++]=ni->c;
1233 #else
1234                 SET_REF(coord_buffer[coord_count], ref);
1235                 coord_count++;
1236 #endif
1237                 ni->ref_way++;
1238         } else {
1239                 len=strlen(p);
1240                 if (len > 0 && p[len-1]=='\n')
1241                         p[len-1]='\0';
1242                 fprintf(stderr,"WARNING: way %d: node %d not found (%s)\n",wayid,ref,p);
1243         }
1244         if (coord_count > 65536) {
1245                 fprintf(stderr,"ERROR: Overflow\n");
1246                 exit(1);
1247         }
1248 }
1249
1250 static int
1251 parse_nd(char *p)
1252 {
1253         char ref_buffer[BUFFER_SIZE];
1254         if (!xml_get_attribute(p, "ref", ref_buffer, BUFFER_SIZE))
1255                 return 0;
1256         add_nd(p, atoi(ref_buffer));
1257         return 1;
1258 }
1259
1260
1261 static void
1262 save_buffer(char *filename, struct buffer *b)
1263 {
1264         FILE *f;
1265         f=fopen(filename,"wb+");
1266         fwrite(b->base, b->size, 1, f);
1267         fclose(f);
1268 }
1269
1270 static void
1271 load_buffer(char *filename, struct buffer *b)
1272 {
1273         FILE *f;
1274         if (b->base)
1275                 free(b->base);
1276         b->malloced=0;
1277         f=fopen(filename,"rb");
1278         fseek(f, 0, SEEK_END);
1279         b->size=b->malloced=ftell(f);
1280         fprintf(stderr,"reading %d bytes from %s\n", b->size, filename);
1281         fseek(f, 0, SEEK_SET);
1282         b->base=malloc(b->size);
1283         assert(b->base != NULL);
1284         fread(b->base, b->size, 1, f);
1285         fclose(f);
1286 }
1287
1288 static int
1289 phase1(FILE *in, FILE *out_ways, FILE *out_nodes)
1290 {
1291         int size=4096;
1292         char buffer[size];
1293         char *p;
1294         sig_alrm(0);
1295         while (fgets(buffer, size, in)) {
1296                 p=strchr(buffer,'<');
1297                 if (! p) {
1298                         fprintf(stderr,"WARNING: wrong line %s\n", buffer);
1299                         continue;
1300                 }
1301                 if (!strncmp(p, "<?xml ",6)) {
1302                 } else if (!strncmp(p, "<osm ",5)) {
1303                 } else if (!strncmp(p, "<bound ",7)) {
1304                 } else if (!strncmp(p, "<node ",6)) {
1305                         if (!parse_node(p))
1306                                 fprintf(stderr,"WARNING: failed to parse %s\n", buffer);
1307                         in_node=1;
1308                         processed_nodes++;
1309                 } else if (!strncmp(p, "<tag ",5)) {
1310                         if (!parse_tag(p))
1311                                 fprintf(stderr,"WARNING: failed to parse %s\n", buffer);
1312                 } else if (!strncmp(p, "<way ",5)) {
1313                         in_way=1;
1314                         if (!parse_way(p))
1315                                 fprintf(stderr,"WARNING: failed to parse %s\n", buffer);
1316                         processed_ways++;
1317                 } else if (!strncmp(p, "<nd ",4)) {
1318                         if (!parse_nd(p))
1319                                 fprintf(stderr,"WARNING: failed to parse %s\n", buffer);
1320                 } else if (!strncmp(p, "<relation ",10)) {
1321                         in_relation=1;
1322                         if (!parse_relation(p))
1323                                 fprintf(stderr,"WARNING: failed to parse %s\n", buffer);
1324                         processed_relations++;
1325                 } else if (!strncmp(p, "<member ",8)) {
1326                 } else if (!strncmp(p, "</node>",7)) {
1327                         in_node=0;
1328                         end_node(out_nodes);
1329                 } else if (!strncmp(p, "</way>",6)) {
1330                         in_way=0;
1331                         end_way(out_ways);
1332                 } else if (!strncmp(p, "</relation>",11)) {
1333                         in_relation=0;
1334                 } else if (!strncmp(p, "</osm>",6)) {
1335                 } else {
1336                         fprintf(stderr,"WARNING: unknown tag in %s\n", buffer);
1337                 }
1338         }
1339         sig_alrm(0);
1340         alarm(0);
1341         return 1;
1342 }
1343
1344 #ifdef HAVE_POSTGRESQL
1345 static int
1346 phase1_db(char *dbstr, FILE *out_ways, FILE *out_nodes)
1347 {
1348         PGconn *conn;
1349         PGresult *res,*node,*way,*tag;
1350         int count,tagged,i,j,k;
1351         long min, max, id, tag_id, node_id;
1352         char query[256];
1353
1354         sig_alrm(0);
1355         conn=PQconnectdb(dbstr);
1356         if (! conn) {
1357                 fprintf(stderr,"Failed to connect to database with '%s'\n",dbstr);
1358                 exit(1);
1359         }
1360         res=PQexec(conn, "begin");
1361         if (! res) {
1362                 fprintf(stderr, "Cannot begin transaction: %s\n", PQerrorMessage(conn));
1363                 PQclear(res);
1364                 exit(1);
1365         }
1366         res=PQexec(conn, "set transaction isolation level serializable");
1367         if (! res) {
1368                 fprintf(stderr, "Cannot set isolation level: %s\n", PQerrorMessage(conn));
1369                 PQclear(res);
1370                 exit(1);
1371         }
1372         res=PQexec(conn, "declare node cursor for select id,x(coordinate),y(coordinate) from node order by id");
1373         if (! res) {
1374                 fprintf(stderr, "Cannot setup cursor for nodes: %s\n", PQerrorMessage(conn));
1375                 PQclear(res);
1376                 exit(1);
1377         }
1378         res=PQexec(conn, "declare way cursor for select id from way order by id");
1379         if (! res) {
1380                 fprintf(stderr, "Cannot setup cursor for nodes: %s\n", PQerrorMessage(conn));
1381                 PQclear(res);
1382                 exit(1);
1383         }
1384         res=PQexec(conn, "declare relation cursor for select id from relation order by id");
1385         if (! res) {
1386                 fprintf(stderr, "Cannot setup cursor for nodes: %s\n", PQerrorMessage(conn));
1387                 PQclear(res);
1388                 exit(1);
1389         }
1390         for (;;) {
1391                 node=PQexec(conn, "fetch 100000 from node");
1392                 if (! node) {
1393                         fprintf(stderr, "Cannot setup cursor for nodes: %s\n", PQerrorMessage(conn));
1394                         PQclear(node);
1395                         exit(1);
1396                 }
1397                 count=PQntuples(node);
1398                 if (! count)
1399                         break;
1400                 min=atol(PQgetvalue(node, 0, 0));
1401                 max=atol(PQgetvalue(node, count-1, 0));
1402                 sprintf(query,"select node_id,name,value from node_tag where node_id >= %ld and node_id <= %ld order by node_id", min, max);
1403                 tag=PQexec(conn, query);
1404                 if (! tag) {
1405                         fprintf(stderr, "Cannot query node_tag: %s\n", PQerrorMessage(conn));
1406                         exit(1);
1407                 }
1408                 j=0;
1409                 for (i = 0 ; i < count ; i++) {
1410                         id=atol(PQgetvalue(node, i, 0));
1411                         add_node(id, atof(PQgetvalue(node, i, 1)), atof(PQgetvalue(node, i, 2)));
1412                         tagged=0;
1413                         in_node=1;
1414                         processed_nodes++;
1415                         while (j < PQntuples(tag)) {
1416                                 tag_id=atol(PQgetvalue(tag, j, 0));
1417                                 if (tag_id == id) {
1418                                         add_tag(PQgetvalue(tag, j, 1), PQgetvalue(tag, j, 2));
1419                                         tagged=1;
1420                                         j++;
1421                                 }
1422                                 if (tag_id < id)
1423                                         j++;
1424                                 if (tag_id > id)
1425                                         break;
1426                         }
1427                         if (tagged)
1428                                 end_node(out_nodes);
1429                         in_node=0;
1430                 }
1431                 PQclear(tag);
1432                 PQclear(node);
1433         }
1434         for (;;) {
1435                 way=PQexec(conn, "fetch 100000 from way");
1436                 if (! way) {
1437                         fprintf(stderr, "Cannot setup cursor for ways: %s\n", PQerrorMessage(conn));
1438                         PQclear(node);
1439                         exit(1);
1440                 }
1441                 count=PQntuples(way);
1442                 if (! count)
1443                         break;
1444                 min=atol(PQgetvalue(way, 0, 0));
1445                 max=atol(PQgetvalue(way, count-1, 0));
1446                 sprintf(query,"select way_id,node_id from way_node where way_id >= %ld and way_id <= %ld order by way_id,sequence_id", min, max);
1447                 node=PQexec(conn, query);
1448                 if (! node) {
1449                         fprintf(stderr, "Cannot query way_node: %s\n", PQerrorMessage(conn));
1450                         exit(1);
1451                 }
1452                 sprintf(query,"select way_id,name,value from way_tag where way_id >= %ld and way_id <= %ld order by way_id", min, max);
1453                 tag=PQexec(conn, query);
1454                 if (! tag) {
1455                         fprintf(stderr, "Cannot query way_tag: %s\n", PQerrorMessage(conn));
1456                         exit(1);
1457                 }
1458                 j=0;
1459                 k=0;
1460                 for (i = 0 ; i < count ; i++) {
1461                         id=atol(PQgetvalue(way, i, 0));
1462                         add_way(id);
1463                         tagged=0;
1464                         in_way=1;
1465                         processed_ways++;
1466                         while (k < PQntuples(node)) {
1467                                 node_id=atol(PQgetvalue(node, k, 0));
1468                                 if (node_id == id) {
1469                                         add_nd("",atol(PQgetvalue(node, k, 1)));
1470                                         tagged=1;
1471                                         k++;
1472                                 }
1473                                 if (node_id < id)
1474                                         k++;
1475                                 if (node_id > id)
1476                                         break;
1477                         }
1478                         while (j < PQntuples(tag)) {
1479                                 tag_id=atol(PQgetvalue(tag, j, 0));
1480                                 if (tag_id == id) {
1481                                         add_tag(PQgetvalue(tag, j, 1), PQgetvalue(tag, j, 2));
1482                                         tagged=1;
1483                                         j++;
1484                                 }
1485                                 if (tag_id < id)
1486                                         j++;
1487                                 if (tag_id > id)
1488                                         break;
1489                         }
1490                         if (tagged)
1491                                 end_way(out_ways);
1492                         in_way=0;
1493                 }
1494                 PQclear(tag);
1495                 PQclear(node);
1496                 PQclear(way);
1497         }
1498
1499         res=PQexec(conn, "commit");
1500         if (! res) {
1501                 fprintf(stderr, "Cannot commit transaction: %s\n", PQerrorMessage(conn));
1502                 PQclear(res);
1503                 exit(1);
1504         }
1505         sig_alrm(0);
1506         alarm(0);
1507         return 1;
1508 }
1509 #endif
1510
1511 static char buffer[200000];
1512
1513 static void
1514 phase1_map(struct map *map, FILE *out_ways, FILE *out_nodes)
1515 {
1516         struct map_rect *mr=map_rect_new(map, NULL);
1517         struct item *item;
1518         int count,max=16384;
1519         struct coord ca[max];
1520         struct item_bin ib;
1521         struct attr attr;
1522         FILE *file;
1523
1524         while ((item = map_rect_get_item(mr))) {
1525                 count=item_coord_get(item, ca, item->type < type_line ? 1: max);
1526                 item_buffer_set_type(buffer, item->type);
1527                 item_buffer_add_coord(buffer, ca, count);
1528                 while (item_attr_get(item, attr_any, &attr)) {
1529                         item_buffer_add_attr(buffer, &attr);
1530                 }
1531                 if (item->type >= type_line)
1532                         item_buffer_write(buffer, out_ways);
1533                 else
1534                         item_buffer_write(buffer, out_nodes);
1535         }
1536         map_rect_destroy(mr);
1537 }
1538
1539
1540 int bytes_read=0;
1541
1542 static struct item_bin *
1543 read_item(FILE *in)
1544 {
1545         struct item_bin *ib=(struct item_bin *) buffer;
1546         int r,s;
1547         r=fread(ib, sizeof(*ib), 1, in);
1548         if (r != 1)
1549                 return NULL;
1550         bytes_read+=r;
1551         assert((ib->len+1)*4 < sizeof(buffer));
1552         s=(ib->len+1)*4-sizeof(*ib);
1553         r=fread(ib+1, s, 1, in);
1554         if (r != 1)
1555                 return NULL;
1556         bytes_read+=r;
1557         return ib;
1558 }
1559
1560 static void
1561 bbox_extend(struct coord *c, struct rect *r)
1562 {
1563         if (c->x < r->l.x)
1564                 r->l.x=c->x;
1565         if (c->y < r->l.y)
1566                 r->l.y=c->y;
1567         if (c->x > r->h.x)
1568                 r->h.x=c->x;
1569         if (c->y > r->h.y)
1570                 r->h.y=c->y;
1571 }
1572
1573 static void
1574 bbox(struct coord *c, int count, struct rect *r)
1575 {
1576         if (! count)
1577                 return;
1578         r->l=*c;
1579         r->h=*c;
1580         while (--count) {
1581                 c++;
1582                 bbox_extend(c, r);
1583         }
1584 }
1585
1586 static int
1587 contains_bbox(int xl, int yl, int xh, int yh, struct rect *r)
1588 {
1589         if (r->h.x < xl || r->h.x > xh) {
1590                 return 0;
1591         }
1592         if (r->l.x > xh || r->l.x < xl) {
1593                 return 0;
1594         }
1595         if (r->h.y < yl || r->h.y > yh) {
1596                 return 0;
1597         }
1598         if (r->l.y > yh || r->l.y < yl) {
1599                 return 0;
1600         }
1601         return 1;
1602
1603 }
1604 struct rect world_bbox = {
1605         { -20000000, -20000000},
1606         {  20000000,  20000000},
1607 };
1608
1609 static void
1610 tile(struct rect *r, char *ret, int max)
1611 {
1612         int x0,x1,x2,x3,x4;
1613         int y0,y1,y2,y3,y4;
1614         int i;
1615         x0=world_bbox.l.x;
1616         y0=world_bbox.l.y;
1617         x4=world_bbox.h.x;
1618         y4=world_bbox.h.y;
1619         for (i = 0 ; i < max ; i++) {
1620                 x2=(x0+x4)/2;
1621                 y2=(y0+y4)/2;
1622                 x1=(x0+x2)/2;
1623                 y1=(y0+y2)/2;
1624                 x3=(x2+x4)/2;
1625                 y3=(y2+y4)/2;
1626                 if (     contains_bbox(x0,y0,x2,y2,r)) {
1627                         strcat(ret,"d");
1628                         x4=x2;
1629                         y4=y2;
1630                 } else if (contains_bbox(x2,y0,x4,y2,r)) {
1631                         strcat(ret,"c");
1632                         x0=x2;
1633                         y4=y2;
1634                 } else if (contains_bbox(x0,y2,x2,y4,r)) {
1635                         strcat(ret,"b");
1636                         x4=x2;
1637                         y0=y2;
1638                 } else if (contains_bbox(x2,y2,x4,y4,r)) {
1639                         strcat(ret,"a");
1640                         x0=x2;
1641                         y0=y2;
1642                 } else
1643                         return;
1644         }
1645 }
1646
1647 static void
1648 tile_bbox(char *tile, struct rect *r)
1649 {
1650         struct coord c;
1651         *r=world_bbox;
1652         while (*tile) {
1653                 c.x=(r->l.x+r->h.x)/2;
1654                 c.y=(r->l.y+r->h.y)/2;
1655                 switch (*tile) {
1656                 case 'a':
1657                         r->l.x=c.x;
1658                         r->l.y=c.y;
1659                         break;
1660                 case 'b':
1661                         r->h.x=c.x;
1662                         r->l.y=c.y;
1663                         break;
1664                 case 'c':
1665                         r->l.x=c.x;
1666                         r->h.y=c.y;
1667                         break;
1668                 case 'd':
1669                         r->h.x=c.x;
1670                         r->h.y=c.y;
1671                         break;
1672                 }
1673                 tile++;
1674         }
1675 }
1676
1677 GHashTable *tile_hash;
1678 GHashTable *tile_hash2;
1679
1680 static void
1681 tile_extend(char *tile, struct item_bin *ib, GList **tiles_list)
1682 {
1683         struct tile_head *th=NULL;
1684         if (debug_tile(tile))
1685                 fprintf(stderr,"Tile:Writing %d bytes to '%s' (%p,%p)\n", (ib->len+1)*4, tile, g_hash_table_lookup(tile_hash, tile), tile_hash2 ? g_hash_table_lookup(tile_hash2, tile) : NULL);
1686         if (tile_hash2)
1687                 th=g_hash_table_lookup(tile_hash2, tile);
1688         if (!th)
1689                 th=g_hash_table_lookup(tile_hash, tile);
1690         if (! th) {
1691                 th=malloc(sizeof(struct tile_head)+ sizeof( char* ) );
1692                 assert(th != NULL);
1693                 // strcpy(th->subtiles, tile);
1694                 th->num_subtiles=1;
1695                 th->total_size=0;
1696                 th->total_size_used=0;
1697                 th->zipnum=0;
1698                 th->zip_data=NULL;
1699                 th->name=string_hash_lookup(tile);
1700                 *th_get_subtile( th, 0 ) = th->name;
1701
1702                 if (tile_hash2)
1703                         g_hash_table_insert(tile_hash2, string_hash_lookup( th->name ), th);
1704                 if (tiles_list)
1705                         *tiles_list=g_list_append(*tiles_list, string_hash_lookup( th->name ) );
1706                 processed_tiles++;
1707                 if (debug_tile(tile))
1708                         fprintf(stderr,"new '%s'\n", tile);
1709         }
1710         th->total_size+=ib->len*4+4;
1711         if (debug_tile(tile))
1712                 fprintf(stderr,"New total size of %s(%p):%d\n", th->name, th, th->total_size);
1713         g_hash_table_insert(tile_hash, string_hash_lookup( th->name ), th);
1714 }
1715
1716 static int
1717 tile_data_size(char *tile)
1718 {
1719         struct tile_head *th;
1720         th=g_hash_table_lookup(tile_hash, tile);
1721         if (! th)
1722                 return 0;
1723         return th->total_size;
1724 }
1725
1726 static int
1727 merge_tile(char *base, char *sub)
1728 {
1729         struct tile_head *thb, *ths;
1730         thb=g_hash_table_lookup(tile_hash, base);
1731         ths=g_hash_table_lookup(tile_hash, sub);
1732         if (! ths)
1733                 return 0;
1734         if (debug_tile(base) || debug_tile(sub))
1735                 fprintf(stderr,"merging '%s'(%p) (%d) with '%s'(%p) (%d)\n", base, thb, thb ? thb->total_size : 0, sub, ths, ths->total_size);
1736         if (! thb) {
1737                 thb=ths;
1738                 g_hash_table_remove(tile_hash, sub);
1739                 thb->name=string_hash_lookup(base);
1740                 g_hash_table_insert(tile_hash, string_hash_lookup( thb->name ), thb);
1741
1742         } else {
1743                 thb=realloc(thb, sizeof(struct tile_head)+( ths->num_subtiles+thb->num_subtiles ) * sizeof( char*) );
1744                 assert(thb != NULL);
1745                 memcpy( th_get_subtile( thb, thb->num_subtiles ), th_get_subtile( ths, 0 ), ths->num_subtiles * sizeof( char*) );
1746                 thb->num_subtiles+=ths->num_subtiles;
1747                 thb->total_size+=ths->total_size;
1748                 g_hash_table_insert(tile_hash, string_hash_lookup( thb->name ), thb);
1749                 g_hash_table_remove(tile_hash, sub);
1750                 g_free(ths);
1751         }
1752         return 1;
1753 }
1754
1755
1756 static void
1757 get_tiles_list_func(char *key, struct tile_head *th, GList **list)
1758 {
1759         *list=g_list_prepend(*list, key);
1760 }
1761
1762 static GList *
1763 get_tiles_list(void)
1764 {
1765         GList *ret=NULL;
1766         g_hash_table_foreach(tile_hash, (GHFunc)get_tiles_list_func, &ret);
1767         return ret;
1768 }
1769
1770 #if 0
1771 static void
1772 write_tile(char *key, struct tile_head *th, gpointer dummy)
1773 {
1774         FILE *f;
1775         char buffer[1024];
1776         fprintf(stderr,"DEBUG: Writing %s\n", key);
1777         strcpy(buffer,"tiles/");
1778         strcat(buffer,key);
1779 #if 0
1780         strcat(buffer,".bin");
1781 #endif
1782         f=fopen(buffer, "wb+");
1783         while (th) {
1784                 fwrite(th->data, th->size, 1, f);
1785                 th=th->next;
1786         }
1787         fclose(f);
1788 }
1789 #endif
1790
1791 static void
1792 write_item(char *tile, struct item_bin *ib)
1793 {
1794         struct tile_head *th;
1795         int size;
1796
1797         th=g_hash_table_lookup(tile_hash2, tile);
1798         if (! th)
1799                 th=g_hash_table_lookup(tile_hash, tile);
1800         if (th) {
1801                 if (th->process != 0 && th->process != 1) {
1802                         fprintf(stderr,"error with tile '%s' of length %d\n", tile, strlen(tile));
1803                         abort();
1804                 }
1805                 if (! th->process)
1806                         return;
1807                 if (debug_tile(tile))
1808                         fprintf(stderr,"Data:Writing %d bytes to '%s' (%p,%p)\n", (ib->len+1)*4, tile, g_hash_table_lookup(tile_hash, tile), tile_hash2 ? g_hash_table_lookup(tile_hash2, tile) : NULL);
1809                 size=(ib->len+1)*4;
1810                 if (th->total_size_used+size > th->total_size) {
1811                         fprintf(stderr,"Overflow in tile %s (used %d max %d item %d)\n", tile, th->total_size_used, th->total_size, size);
1812                         exit(1);
1813                         return;
1814                 }
1815                 memcpy(th->zip_data+th->total_size_used, ib, size);
1816                 th->total_size_used+=size;
1817         } else {
1818                 fprintf(stderr,"no tile hash found for %s\n", tile);
1819                 exit(1);
1820         }
1821 }
1822
1823 static void
1824 write_item_part(FILE *out, struct item_bin *orig, int first, int last)
1825 {
1826         struct item_bin new;
1827         struct coord *c=(struct coord *)(orig+1);
1828         char *attr=(char *)(c+orig->clen/2);
1829         int attr_len=orig->len-orig->clen-2;
1830         processed_ways++;
1831         new.type=orig->type;
1832         new.clen=(last-first+1)*2;
1833         new.len=new.clen+attr_len+2;
1834 #if 0
1835         fprintf(stderr,"first %d last %d type 0x%x len %d clen %d attr_len %d\n", first, last, new.type, new.len, new.clen, attr_len);
1836 #endif
1837         fwrite(&new, sizeof(new), 1, out);
1838         fwrite(c+first, new.clen*4, 1, out);
1839         fwrite(attr, attr_len*4, 1, out);
1840 }
1841
1842 static int
1843 phase2(FILE *in, FILE *out)
1844 {
1845         struct coord *c;
1846         int i,ccount,last,ndref;
1847         struct item_bin *ib;
1848         struct node_item *ni;
1849
1850         processed_nodes=processed_nodes_out=processed_ways=processed_relations=processed_tiles=0;
1851         sig_alrm(0);
1852         while ((ib=read_item(in))) {
1853 #if 0
1854                 fprintf(stderr,"type 0x%x len %d clen %d\n", ib->type, ib->len, ib->clen);
1855 #endif
1856                 ccount=ib->clen/2;
1857                 c=(struct coord *)(ib+1);
1858                 last=0;
1859                 for (i = 0 ; i < ccount ; i++) {
1860                         if (IS_REF(c[i])) {
1861                                 ndref=REF(c[i]);
1862                                 ni=node_item_get(ndref);
1863 #if 0
1864                                 fprintf(stderr,"ni=%p\n", ni);
1865 #endif
1866                                 c[i]=ni->c;
1867                                 if (ni->ref_way > 1 && i != 0 && i != ccount-1 && item_is_street(*ib)) {
1868                                         write_item_part(out, ib, last, i);
1869                                         last=i;
1870                                 }
1871                         }
1872                 }
1873                 write_item_part(out, ib, last, ccount-1);
1874         }
1875         sig_alrm(0);
1876         alarm(0);
1877         return 0;
1878 }
1879
1880 static void
1881 phase34_process_file(int phase, FILE *in)
1882 {
1883         struct item_bin *ib;
1884         struct rect r;
1885         char buffer[1024];
1886         int max;
1887
1888         while ((ib=read_item(in))) {
1889                 if (ib->type < 0x80000000)
1890                         processed_nodes++;
1891                 else
1892                         processed_ways++;
1893                 bbox((struct coord *)(ib+1), ib->clen/2, &r);
1894                 buffer[0]='\0';
1895                 max=14;
1896                 if (ib->type == type_street_n_lanes || ib->type == type_highway_city || ib->type == type_highway_land || ib->type == type_ramp)
1897                         max=8;
1898                 if (ib->type == type_street_3_city || ib->type == type_street_4_city || ib->type == type_street_3_land || ib->type == type_street_4_land)
1899                         max=12;
1900
1901                 tile(&r, buffer, max);
1902 #if 0
1903                 fprintf(stderr,"%s\n", buffer);
1904 #endif
1905                 if (phase == 3)
1906                         tile_extend(buffer, ib, NULL);
1907                 else
1908                         write_item(buffer, ib);
1909         }
1910 }
1911
1912 struct index_item {
1913         struct item_bin item;
1914         struct rect r;
1915         struct attr_bin attr_order;
1916         short min;
1917         short max;
1918         struct attr_bin attr_zipfile_ref;
1919         int zipfile_ref;
1920 };
1921
1922 static void
1923 index_submap_add(int phase, struct tile_head *th, GList **tiles_list)
1924 {
1925         struct index_item ii;
1926         int len=strlen(th->name);
1927         char index_tile[len+1];
1928
1929         ii.min=(len > 4) ? len-4 : 0;
1930         ii.max=255;
1931         strcpy(index_tile, th->name);
1932         if (len > 6)
1933                 len=6;
1934         else
1935                 len=0;
1936         index_tile[len]=0;
1937         tile_bbox(th->name, &ii.r);
1938
1939         ii.item.len=sizeof(ii)/4-1;
1940         ii.item.type=type_submap;
1941         ii.item.clen=4;
1942
1943         ii.attr_order.len=2;
1944         ii.attr_order.type=attr_order;
1945
1946         ii.attr_zipfile_ref.len=2;
1947         ii.attr_zipfile_ref.type=attr_zipfile_ref;
1948         ii.zipfile_ref=th->zipnum;
1949
1950         if (phase == 3)
1951                 tile_extend(index_tile, (struct item_bin *)&ii, tiles_list);
1952         else
1953                 write_item(index_tile, (struct item_bin *)&ii);
1954 #if 0
1955         unsigned int *c=(unsigned int *)&ii;
1956         int i;
1957         for (i = 0 ; i < sizeof(ii)/4 ; i++) {
1958                 fprintf(stderr,"%08x ", c[i]);
1959         }
1960         fprintf(stderr,"\n");
1961 #endif
1962 }
1963
1964 static int
1965 add_tile_hash(struct tile_head *th)
1966 {
1967         int idx,len,maxnamelen=0;
1968         char **data;
1969
1970 #if 0
1971         g_hash_table_insert(tile_hash2, string_hash_lookup( th->name ), th);
1972 #endif
1973         for( idx = 0; idx < th->num_subtiles; idx++ ) {
1974
1975         data = th_get_subtile( th, idx );
1976
1977                 if (debug_tile(data) || debug_tile(th->name)) {
1978                         fprintf(stderr,"Parent for '%s' is '%s'\n", *data, th->name);
1979                 }
1980
1981                 g_hash_table_insert(tile_hash2, *data, th);
1982
1983                 len = strlen( *data );
1984
1985                 if (len > maxnamelen) {
1986                         maxnamelen=len;
1987                 }
1988         }
1989         return maxnamelen;
1990 }
1991
1992
1993 static int
1994 create_tile_hash(void)
1995 {
1996         struct tile_head *th;
1997         int len,maxnamelen=0;
1998
1999         tile_hash2=g_hash_table_new(g_str_hash, g_str_equal);
2000         th=tile_head_root;
2001         while (th) {
2002                 len=add_tile_hash(th);
2003                 if (len > maxnamelen)
2004                         maxnamelen=len;
2005                 th=th->next;
2006         }
2007         return maxnamelen;
2008 }
2009
2010 static void
2011 create_tile_hash_list(GList *list)
2012 {
2013         GList *next;
2014         struct tile_head *th;
2015
2016         tile_hash2=g_hash_table_new(g_str_hash, g_str_equal);
2017
2018         fprintf(stderr,"list=%p\n", list);
2019         next=g_list_first(list);
2020         while (next) {
2021                 th=g_hash_table_lookup(tile_hash, next->data);
2022                 if (!th) {
2023                         fprintf(stderr,"No tile found for '%s'\n", (char *)(next->data));
2024                 }
2025                 add_tile_hash(th);
2026                 next=g_list_next(next);
2027         }
2028 }
2029
2030 static void
2031 destroy_tile_hash(void)
2032 {
2033         g_hash_table_destroy(tile_hash2);
2034         tile_hash2=NULL;
2035 }
2036
2037 static int zipnum;
2038
2039 static void write_countrydir(int phase, int maxnamelen);
2040
2041 static void
2042 write_tilesdir(int phase, int maxlen, FILE *out)
2043 {
2044         int idx,len;
2045         GList *tiles_list,*next;
2046         char **data;
2047         struct tile_head *th,**last=NULL;
2048         zipnum=0;
2049
2050         tiles_list=get_tiles_list();
2051         if (phase == 3)
2052                 create_tile_hash_list(tiles_list);
2053         next=g_list_first(tiles_list);
2054         last=&tile_head_root;
2055         if (! maxlen) {
2056                 while (next) {
2057                         if (strlen(next->data) > maxlen)
2058                                 maxlen=strlen(next->data);
2059                         next=g_list_next(next);
2060                 }
2061         }
2062         len=maxlen;
2063         while (len >= 0) {
2064 #if 0
2065                 fprintf(stderr,"PROGRESS: collecting tiles with len=%d\n", len);
2066 #endif
2067 #ifdef GENERATE_INDEX
2068                 if (! len)
2069                         write_countrydir(phase, maxlen);
2070 #endif
2071                 next=g_list_first(tiles_list);
2072                 while (next) {
2073                         if (strlen(next->data) == len) {
2074                                 th=g_hash_table_lookup(tile_hash, next->data);
2075                                 if (phase == 3) {
2076                                         *last=th;
2077                                         last=&th->next;
2078                                         th->next=NULL;
2079                                         th->zipnum=zipnum;
2080                                         fprintf(out,"%s:%d",(char *)next->data,th->total_size);
2081
2082                                         for ( idx = 0; idx< th->num_subtiles; idx++ ){
2083                         data= th_get_subtile( th, idx );
2084                                                 fprintf(out,":%s", *data);
2085                                         }
2086
2087                                         fprintf(out,"\n");
2088                                 }
2089                                 if (th->name[0])
2090                                         index_submap_add(phase, th, &tiles_list);
2091                                 zipnum++;
2092                                 processed_tiles++;
2093                         }
2094                         next=g_list_next(next);
2095                 }
2096                 len--;
2097         }
2098 }
2099
2100 static void
2101 merge_tiles(void)
2102 {
2103         struct tile_head *th;
2104         char basetile[1024];
2105         char subtile[1024];
2106         GList *tiles_list_sorted,*last;
2107         int i,i_min,len,size_all,size[5],size_min,work_done;
2108         long long zip_size;
2109
2110         do {
2111                 tiles_list_sorted=get_tiles_list();
2112                 fprintf(stderr,"PROGRESS: sorting %d tiles\n", g_list_length(tiles_list_sorted));
2113                 tiles_list_sorted=g_list_sort(tiles_list_sorted, (GCompareFunc)strcmp);
2114                 fprintf(stderr,"PROGRESS: sorting %d tiles done\n", g_list_length(tiles_list_sorted));
2115                 last=g_list_last(tiles_list_sorted);
2116                 zip_size=0;
2117                 while (last) {
2118                         th=g_hash_table_lookup(tile_hash, last->data);
2119                         zip_size+=th->total_size;
2120                         last=g_list_previous(last);
2121                 }
2122                 fprintf(stderr,"DEBUG: size=%Ld\n", zip_size);
2123                 last=g_list_last(tiles_list_sorted);
2124                 work_done=0;
2125                 while (last) {
2126                         processed_tiles++;
2127                         len=strlen(last->data);
2128                         if (len >= 1) {
2129                                 strcpy(basetile,last->data);
2130                                 basetile[len-1]='\0';
2131                                 strcpy(subtile,last->data);
2132                                 for (i = 0 ; i < 4 ; i++) {
2133                                         subtile[len-1]='a'+i;
2134                                         size[i]=tile_data_size(subtile);
2135                                 }
2136                                 size[4]=tile_data_size(basetile);
2137                                 size_all=size[0]+size[1]+size[2]+size[3]+size[4];
2138                                 if (size_all < 65536 && size_all > 0 && size_all != size[4]) {
2139                                         for (i = 0 ; i < 4 ; i++) {
2140                                                 subtile[len-1]='a'+i;
2141                                                 work_done+=merge_tile(basetile, subtile);
2142                                         }
2143                                 } else {
2144                                         for (;;) {
2145                                                 size_min=size_all;
2146                                                 i_min=-1;
2147                                                 for (i = 0 ; i < 4 ; i++) {
2148                                                         if (size[i] && size[i] < size_min) {
2149                                                                 size_min=size[i];
2150                                                                 i_min=i;
2151                                                         }
2152                                                 }
2153                                                 if (i_min == -1)
2154                                                         break;
2155                                                 if (size[4]+size_min >= 65536)
2156                                                         break;
2157                                                 subtile[len-1]='a'+i_min;
2158                                                 work_done+=merge_tile(basetile, subtile);
2159                                                 size[4]+=size[i_min];
2160                                                 size[i_min]=0;
2161                                         }
2162                                 }
2163                         }
2164                         last=g_list_previous(last);
2165                 }
2166                 g_list_free(tiles_list_sorted);
2167                 fprintf(stderr,"PROGRESS: merged %d tiles\n", work_done);
2168         } while (work_done);
2169 }
2170
2171 struct country_index_item {
2172         struct item_bin item;
2173         struct attr_bin attr_country_id;
2174         int country_id;
2175         struct attr_bin attr_zipfile_ref;
2176         int zipfile_ref;
2177 };
2178
2179 static void
2180 index_country_add(int phase, int country_id, int zipnum)
2181 {
2182         struct country_index_item ii;
2183         char *index_tile="";
2184
2185         ii.item.len=sizeof(ii)/4-1;
2186         ii.item.type=type_countryindex;
2187         ii.item.clen=0;
2188
2189         ii.attr_country_id.len=2;
2190         ii.attr_country_id.type=attr_country_id;
2191         ii.country_id=country_id;
2192
2193         ii.attr_zipfile_ref.len=2;
2194         ii.attr_zipfile_ref.type=attr_zipfile_ref;
2195         ii.zipfile_ref=zipnum;
2196
2197         if (phase == 3)
2198                 tile_extend(index_tile, (struct item_bin *)&ii, NULL);
2199         else
2200                 write_item(index_tile, (struct item_bin *)&ii);
2201 }
2202
2203 #ifdef GENERATE_INDEX
2204 struct aux_tile {
2205         char *name;
2206         char *filename;
2207         int size;
2208 };
2209
2210 static int
2211 add_aux_tile(int phase, char *name, char *filename, int size)
2212 {
2213         struct aux_tile *at;
2214         if (phase == 3) {
2215                 at=g_new0(struct aux_tile, 1);
2216                 at->name=g_strdup(name);
2217                 at->filename=g_strdup(filename);
2218                 at->size=size;
2219                 aux_tile_list=g_list_append(aux_tile_list, at);
2220         }
2221         return zipnum++;
2222 }
2223
2224 static int
2225 write_aux_tiles(FILE *out, FILE *dir_out, int compression_level, int namelen)
2226 {
2227         GList *l=aux_tile_list;
2228         struct aux_tile *at;
2229         char *buffer;
2230         FILE *f;
2231         int count=0;
2232
2233         while (l) {
2234                 at=l->data;
2235                 buffer=malloc(at->size);
2236                 assert(buffer != NULL);
2237                 f=fopen(at->filename,"r");
2238                 assert(f != NULL);
2239                 fread(buffer, at->size, 1, f);
2240                 fclose(f);
2241                 write_zipmember(out, dir_out, at->name, namelen, buffer, at->size, compression_level);
2242                 count++;
2243                 l=g_list_next(l);
2244         }
2245         return count;
2246 }
2247
2248 static void
2249 write_countrydir(int phase, int maxnamelen)
2250 {
2251         int i,zipnum;
2252         int max=11;
2253         char tilename[32];
2254         char searchtile[32];
2255         char filename[32];
2256         struct country_table *co;
2257         for (i = 0 ; i < sizeof(country_table)/sizeof(struct country_table) ; i++) {
2258                 co=&country_table[i];
2259                 if (co->size) {
2260                         tilename[0]='\0';
2261                         tile(&co->r, tilename, max);
2262                         sprintf(searchtile,"%ss%d", tilename, 0);
2263                         sprintf(filename,"country_%d.bin", co->countryid);
2264                         zipnum=add_aux_tile(phase, searchtile, filename, co->size);
2265                         index_country_add(phase,co->countryid,zipnum);
2266                 }
2267         }
2268 }
2269
2270 static void
2271 remove_countryfiles(void)
2272 {
2273         int i;
2274         char filename[32];
2275         struct country_table *co;
2276
2277         for (i = 0 ; i < sizeof(country_table)/sizeof(struct country_table) ; i++) {
2278                 co=&country_table[i];
2279                 if (co->size) {
2280                         sprintf(filename,"country_%d.bin", co->countryid);
2281                         unlink(filename);
2282                 }
2283         }
2284 }
2285 #endif
2286
2287 static int
2288 phase34(int phase, int maxnamelen, FILE *ways_in, FILE *nodes_in, FILE *tilesdir_out)
2289 {
2290
2291         processed_nodes=processed_nodes_out=processed_ways=processed_relations=processed_tiles=0;
2292         bytes_read=0;
2293         sig_alrm(0);
2294         if (phase == 3)
2295                 tile_hash=g_hash_table_new(g_str_hash, g_str_equal);
2296         if (ways_in)
2297                 phase34_process_file(phase, ways_in);
2298         if (nodes_in)
2299                 phase34_process_file(phase, nodes_in);
2300         fprintf(stderr,"read %d bytes\n", bytes_read);
2301         if (phase == 3)
2302                 merge_tiles();
2303         sig_alrm(0);
2304         alarm(0);
2305         write_tilesdir(phase, maxnamelen, tilesdir_out);
2306
2307         return 0;
2308
2309 }
2310
2311 static void
2312 dump_coord(struct coord *c)
2313 {
2314         printf("0x%x 0x%x",c->x, c->y);
2315 }
2316
2317 static void
2318 dump(FILE *in)
2319 {
2320         struct item_bin *ib;
2321         struct coord *c;
2322         struct attr_bin *a;
2323         struct attr attr;
2324         int *attr_start;
2325         int *attr_end;
2326         int i;
2327         char *str;
2328         while ((ib=read_item(in))) {
2329                 c=(struct coord *)(ib+1);
2330                 if (ib->type < type_line) {
2331                         dump_coord(c);
2332                         printf(" ");
2333                 }
2334                 attr_start=(int *)(ib+1)+ib->clen;
2335                 attr_end=(int *)ib+ib->len+1;
2336                 printf("type=%s", item_to_name(ib->type));
2337                 while (attr_start < attr_end) {
2338                         a=(struct attr_bin *)(attr_start);
2339                         attr_start+=a->len+1;
2340                         attr.type=a->type;
2341                         attr_data_set(&attr, (a+1));
2342                         str=attr_to_text(&attr, NULL, 1);
2343                         printf(" %s=\"%s\"", attr_to_name(a->type), str);
2344                         g_free(str);
2345                 }
2346                 printf(" debug=\"length=%d\"", ib->len);
2347                 printf("\n");
2348                 if (ib->type >= type_line) {
2349                         for (i = 0 ; i < ib->clen/2 ; i++) {
2350                                 dump_coord(c+i);
2351                                 printf("\n");
2352                         }
2353
2354                 }
2355         }
2356 }
2357
2358 static int
2359 phase3(FILE *ways_in, FILE *nodes_in, FILE *tilesdir_out)
2360 {
2361         return phase34(3, 0, ways_in, nodes_in, tilesdir_out);
2362 }
2363
2364 static long long zipoffset;
2365 static int zipdir_size;
2366
2367 static int
2368 compress2_int(Byte *dest, uLongf *destLen, const Bytef *source, uLong sourceLen, int level)
2369 {
2370         z_stream stream;
2371         int err;
2372
2373         stream.next_in = (Bytef*)source;
2374         stream.avail_in = (uInt)sourceLen;
2375         stream.next_out = dest;
2376         stream.avail_out = (uInt)*destLen;
2377         if ((uLong)stream.avail_out != *destLen) return Z_BUF_ERROR;
2378
2379         stream.zalloc = (alloc_func)0;
2380         stream.zfree = (free_func)0;
2381         stream.opaque = (voidpf)0;
2382
2383         err = deflateInit2(&stream, level, Z_DEFLATED, -15, 9, Z_DEFAULT_STRATEGY);
2384         if (err != Z_OK) return err;
2385
2386         err = deflate(&stream, Z_FINISH);
2387         if (err != Z_STREAM_END) {
2388                 deflateEnd(&stream);
2389                 return err == Z_OK ? Z_BUF_ERROR : err;
2390         }
2391         *destLen = stream.total_out;
2392
2393         err = deflateEnd(&stream);
2394         return err;
2395 }
2396
2397 static void
2398 write_zipmember(FILE *out, FILE *dir_out, char *name, int filelen, char *data, int data_size, int compression_level)
2399 {
2400         struct zip_lfh lfh = {
2401                 0x04034b50,
2402                 0x0a,
2403                 0x0,
2404                 0x0,
2405                 0xbe2a,
2406                 0x5d37,
2407                 0x0,
2408                 0x0,
2409                 0x0,
2410                 filelen,
2411                 0x0,
2412         };
2413         struct zip_cd cd = {
2414                 0x02014b50,
2415                 0x17,
2416                 0x00,
2417                 0x0a,
2418                 0x00,
2419                 0x0000,
2420                 0x0,
2421                 0xbe2a,
2422                 0x5d37,
2423                 0x0,
2424                 0x0,
2425                 0x0,
2426                 filelen,
2427                 0x0000,
2428                 0x0000,
2429                 0x0000,
2430                 0x0000,
2431                 0x0,
2432                 zipoffset,
2433         };
2434         char filename[filelen+1];
2435         int error,crc,len,comp_size=data_size;
2436         uLongf destlen=data_size+data_size/500+12;
2437         char compbuffer[destlen];
2438
2439         crc=crc32(0, NULL, 0);
2440         crc=crc32(crc, (unsigned char *)data, data_size);
2441         if (compression_level) {
2442                 error=compress2_int((Byte *)compbuffer, &destlen, (Bytef *)data, data_size, compression_level);
2443                 if (error == Z_OK) {
2444                         if (destlen < data_size) {
2445                                 data=compbuffer;
2446                                 comp_size=destlen;
2447                         }
2448                 } else {
2449                         fprintf(stderr,"compress2 returned %d\n", error);
2450                 }
2451         }
2452         lfh.zipcrc=crc;
2453         lfh.zipsize=comp_size;
2454         lfh.zipuncmp=data_size;
2455         lfh.zipmthd=compression_level ? 8:0;
2456         cd.zipccrc=crc;
2457         cd.zipcsiz=comp_size;
2458         cd.zipcunc=data_size;
2459         cd.zipcmthd=compression_level ? 8:0;
2460         strcpy(filename, name);
2461         len=strlen(filename);
2462         while (len < filelen) {
2463                 filename[len++]='_';
2464         }
2465         filename[filelen]='\0';
2466         fwrite(&lfh, sizeof(lfh), 1, out);
2467         fwrite(filename, filelen, 1, out);
2468         fwrite(data, comp_size, 1, out);
2469         zipoffset+=sizeof(lfh)+filelen+comp_size;
2470         fwrite(&cd, sizeof(cd), 1, dir_out);
2471         fwrite(filename, filelen, 1, dir_out);
2472         zipdir_size+=sizeof(cd)+filelen;
2473 }
2474
2475 static int
2476 process_slice(FILE *ways_in, FILE *nodes_in, int size, int maxnamelen, FILE *out, FILE *dir_out, int compression_level)
2477 {
2478         struct tile_head *th;
2479         char *slice_data,*zip_data;
2480         int zipfiles=0;
2481
2482         slice_data=malloc(size);
2483         assert(slice_data != NULL);
2484         zip_data=slice_data;
2485         th=tile_head_root;
2486         while (th) {
2487                 if (th->process) {
2488                         th->zip_data=zip_data;
2489                         zip_data+=th->total_size;
2490                 }
2491                 th=th->next;
2492         }
2493         if (ways_in)
2494                 fseek(ways_in, 0, SEEK_SET);
2495         if (nodes_in)
2496                 fseek(nodes_in, 0, SEEK_SET);
2497         phase34(4, maxnamelen, ways_in, nodes_in, NULL);
2498
2499         th=tile_head_root;
2500         while (th) {
2501                 if (th->process) {
2502 #ifdef GENERATE_INDEX
2503                         if (! strlen(th->name))
2504                                 zipfiles+=write_aux_tiles(out, dir_out, compression_level, maxnamelen);
2505 #endif
2506                         if (th->total_size != th->total_size_used) {
2507                                 fprintf(stderr,"Size error '%s': %d vs %d\n", th->name, th->total_size, th->total_size_used);
2508                                 exit(1);
2509                         } else {
2510                                 if (strlen(th->name))
2511                                         write_zipmember(out, dir_out, th->name, maxnamelen, th->zip_data, th->total_size, compression_level);
2512                                 else {
2513                                         write_zipmember(out, dir_out, "index", sizeof("index")-1, th->zip_data, th->total_size, compression_level);
2514                                 }
2515                                 zipfiles++;
2516                         }
2517                 }
2518                 th=th->next;
2519         }
2520         free(slice_data);
2521
2522         return zipfiles;
2523 }
2524
2525 static void
2526 cat(FILE *in, FILE *out)
2527 {
2528         size_t size;
2529         char buffer[4096];
2530         while ((size=fread(buffer, 1, 4096, in)))
2531                 fwrite(buffer, 1, size, out);
2532 }
2533
2534 static int
2535 phase4(FILE *ways_in, FILE *nodes_in, FILE *out, FILE *dir_out, int compression_level)
2536 {
2537         int slice_size=1024*1024*1024;
2538         int maxnamelen,size,slices;
2539         int zipfiles=0;
2540         struct tile_head *th,*th2;
2541         struct zip_eoc eoc = {
2542                 0x06054b50,
2543                 0x0000,
2544                 0x0000,
2545                 0x0000,
2546                 0x0000,
2547                 0x0,
2548                 0x0,
2549                 0x0,
2550         };
2551
2552         maxnamelen=create_tile_hash();
2553
2554         th=tile_head_root;
2555         size=0;
2556         slices=0;
2557         fprintf(stderr, "Maximum slice size %d\n", slice_size);
2558         while (th) {
2559                 if (size + th->total_size > slice_size) {
2560                         fprintf(stderr,"Slice %d is of size %d\n", slices, size);
2561                         size=0;
2562                         slices++;
2563                 }
2564                 size+=th->total_size;
2565                 th=th->next;
2566         }
2567         if (size)
2568                 fprintf(stderr,"Slice %d is of size %d\n", slices, size);
2569         th=tile_head_root;
2570         size=0;
2571         slices=0;
2572         while (th) {
2573                 th2=tile_head_root;
2574                 while (th2) {
2575                         th2->process=0;
2576                         th2=th2->next;
2577                 }
2578                 size=0;
2579                 while (th && size+th->total_size < slice_size) {
2580                         size+=th->total_size;
2581                         th->process=1;
2582                         th=th->next;
2583                 }
2584                 zipfiles+=process_slice(ways_in, nodes_in, size, maxnamelen, out, dir_out, compression_level);
2585                 slices++;
2586         }
2587         fseek(dir_out, 0, SEEK_SET);
2588         cat(dir_out, out);
2589         eoc.zipenum=zipfiles;
2590         eoc.zipecenn=zipfiles;
2591         eoc.zipecsz=zipdir_size;
2592         eoc.zipeofst=zipoffset;
2593         fwrite(&eoc, sizeof(eoc), 1, out);
2594         sig_alrm(0);
2595         alarm(0);
2596         return 0;
2597 }
2598
2599 static void
2600 usage(FILE *f)
2601 {
2602         /* DEVELOPPERS : don't forget to update the manpage if you modify theses options */
2603         fprintf(f,"\n");
2604         fprintf(f,"osm2navit - parse osm textfile and converts to NavIt binfile format\n\n");
2605         fprintf(f,"Usage :\n");
2606         fprintf(f,"bzcat planet.osm.bz2 | osm2navit mymap.bin\n");
2607         fprintf(f,"Available switches:\n");
2608         fprintf(f,"-h (--help)              : this screen\n");
2609         fprintf(f,"-N (--nodes-only)        : process only nodes\n");
2610         fprintf(f,"-W (--ways-only)         : process only ways\n");
2611         fprintf(f,"-a (--attr-debug-level)  : control which data is included in the debug attribute\n");
2612         fprintf(f,"-c (--dump-coordinates)  : dump coordinates after phase 1\n");
2613 #ifdef HAVE_POSTGRESQL
2614         fprintf(f,"-d (--db)                : get osm data out of a postgresql database with osm simple scheme and given connect string\n");
2615 #endif
2616         fprintf(f,"-e (--end)               : end at specified phase\n");
2617         fprintf(f,"-k (--keep-tmpfiles)     : do not delete tmp files after processing. useful to reuse them\n\n");
2618         fprintf(f,"-o (--coverage)          : map every street to item overage\n");
2619         fprintf(f,"-s (--start)             : start at specified phase\n");
2620         fprintf(f,"-i (--input-file)        : specify the input file name (OSM), overrules default stdin\n");
2621         fprintf(f,"-w (--dedupe-ways)       : ensure no duplicate ways or nodes. useful when using several input files\n");
2622         fprintf(f,"-z (--compression-level) : set the compression level\n");
2623         exit(1);
2624 }
2625
2626 static void
2627 process_binfile(FILE *in, FILE *out)
2628 {
2629         struct item_bin *ib;
2630         while ((ib=read_item(in))) {
2631                 fwrite(ib, (ib->len+1)*4, 1, out);
2632         }
2633 }
2634
2635 int main(int argc, char **argv)
2636 {
2637         FILE *ways=NULL,*ways_split=NULL,*nodes=NULL,*tilesdir,*zipdir,*res;
2638         char *map=g_strdup(attrmap);
2639         int i,c,start=1,end=4,dump_coordinates=0;
2640         int keep_tmpfiles=0;
2641         int process_nodes=1, process_ways=1;
2642         int compression_level=9;
2643         int output=0;
2644         int input=0;
2645         char *result,*dbstr=NULL;
2646         FILE* input_file = stdin;
2647         struct plugins *plugins=NULL;
2648         struct attr **attrs;
2649         struct map *map_handle=NULL;
2650
2651         while (1) {
2652 #if 0
2653                 int this_option_optind = optind ? optind : 1;
2654 #endif
2655                 int option_index = 0;
2656                 static struct option long_options[] = {
2657                         {"attr-debug-level", 1, 0, 'a'},
2658                         {"binfile", 0, 0, 'b'},
2659                         {"compression-level", 1, 0, 'z'},
2660                         {"coverage", 0, 0, 'o'},
2661 #ifdef HAVE_POSTGRESQL
2662                         {"db", 1, 0, 'd'},
2663 #endif
2664                         {"dedupe-ways", 0, 0, 'w'},
2665                         {"dump", 0, 0, 'D'},
2666                         {"dump-coordinates", 0, 0, 'c'},
2667                         {"end", 1, 0, 'e'},
2668                         {"help", 0, 0, 'h'},
2669                         {"keep-tmpfiles", 0, 0, 'k'},
2670                         {"nodes-only", 0, 0, 'N'},
2671                         {"map", 1, 0, 'm'},
2672                         {"plugin", 1, 0, 'p'},
2673                         {"start", 1, 0, 's'},
2674                         {"input-file", 1, 0, 'i'},
2675                         {"ignore-unknown", 0, 0, 'n'},
2676                         {"ways-only", 0, 0, 'W'},
2677                         {0, 0, 0, 0}
2678                 };
2679                 c = getopt_long (argc, argv, "DNWa:bc"
2680 #ifdef HAVE_POSTGRESQL
2681                                               "d:"
2682 #endif
2683                                               "e:hi:knm:p:s:w", long_options, &option_index);
2684                 if (c == -1)
2685                         break;
2686                 switch (c) {
2687                 case 'D':
2688                         output=1;
2689                         break;
2690                 case 'N':
2691                         process_ways=0;
2692                         break;
2693                 case 'W':
2694                         process_nodes=0;
2695                         break;
2696                 case 'a':
2697                         attr_debug_level=atoi(optarg);
2698                         break;
2699                 case 'b':
2700                         input=1;
2701                         break;
2702                 case 'c':
2703                         dump_coordinates=1;
2704                         break;
2705 #ifdef HAVE_POSTGRESQL
2706                 case 'd':
2707                         dbstr=optarg;
2708                         break;
2709 #endif
2710                 case 'e':
2711                         end=atoi(optarg);
2712                         break;
2713                 case 'h':
2714                         usage(stdout);
2715                         break;
2716                 case 'm':
2717                         attrs=(struct attr*[]){
2718                                 &(struct attr){attr_type,{"textfile"}},
2719                                 &(struct attr){attr_data,{"bookmark.txt"}},
2720                                 NULL};
2721                         map_handle=map_new(attrs);
2722                         fprintf(stderr,"optarg=%s\n", optarg);
2723                         break;
2724                 case 'n':
2725                         fprintf(stderr,"I will IGNORE unknown types\n");
2726                         ignore_unkown=1;
2727                         break;
2728                 case 'k':
2729                         fprintf(stderr,"I will KEEP tmp files\n");
2730                         keep_tmpfiles=1;
2731                         break;
2732                 case 'o':
2733                         coverage=1;
2734                         break;
2735                 case 'p':
2736                         if (! plugins)
2737                                 plugins=plugins_new();
2738                         fprintf(stderr,"optarg=%s\n",optarg);
2739                         attrs=(struct attr*[]){&(struct attr){attr_path,{optarg}},NULL};
2740                         plugins_add_path(plugins, attrs);
2741                         break;
2742                 case 's':
2743                         start=atoi(optarg);
2744                         break;
2745                 case 'w':
2746                         dedupe_ways_hash=g_hash_table_new(NULL, NULL);
2747                         break;
2748                 case 'i':
2749                         input_file = fopen( optarg, "r" );
2750                         if ( input_file ==  NULL )
2751                         {
2752                             fprintf( stderr, "\nInput file (%s) not found\n", optarg );
2753                             exit( -1 );
2754                         }
2755                         break;
2756                 case 'z':
2757                         compression_level=atoi(optarg);
2758                         break;
2759                 case '?':
2760                         usage(stderr);
2761                         break;
2762                 default:
2763                         fprintf(stderr,"c=%d\n", c);
2764                 }
2765
2766         }
2767         if (optind != argc-(output == 1 ? 0:1))
2768                 usage(stderr);
2769         if (plugins)
2770                 plugins_init(plugins);
2771         result=argv[optind];
2772         build_attrmap(map);
2773 #ifdef GENERATE_INDEX
2774         build_countrytable();
2775 #endif
2776
2777
2778         if (input == 0) {
2779         if (start == 1) {
2780                 if (process_ways)
2781                         ways=fopen("ways.tmp","wb+");
2782                 if (process_nodes)
2783                         nodes=fopen("nodes.tmp","wb+");
2784                 phase=1;
2785                 fprintf(stderr,"PROGRESS: Phase 1: collecting data\n");
2786 #ifdef HAVE_POSTGRESQL
2787                 if (dbstr)
2788                         phase1_db(dbstr,ways,nodes);
2789                 else
2790 #endif
2791                 if (map_handle) {
2792                         phase1_map(map_handle,ways,nodes);
2793                         map_destroy(map_handle);
2794                 }
2795                 else
2796                         phase1(input_file,ways,nodes);
2797                 if (ways)
2798                         fclose(ways);
2799                 if (nodes)
2800                         fclose(nodes);
2801 #ifdef GENERATE_INDEX
2802                 fprintf(stderr,"PROGRESS: Phase 1: sorting countries\n");
2803                 sort_countries();
2804 #endif
2805         }
2806         if (end == 1 || dump_coordinates)
2807                 save_buffer("coords.tmp",&node_buffer);
2808         if (end == 1)
2809                 exit(0);
2810         if (start == 2)
2811                 load_buffer("coords.tmp",&node_buffer);
2812         if (start <= 2) {
2813                 if (process_ways) {
2814                         ways=fopen("ways.tmp","rb");
2815                         ways_split=fopen("ways_split.tmp","wb+");
2816                         phase=2;
2817                         fprintf(stderr,"PROGRESS: Phase 2: finding intersections\n");
2818                         phase2(ways,ways_split);
2819                         fclose(ways_split);
2820                         fclose(ways);
2821                         if(!keep_tmpfiles)
2822                                 remove("ways.tmp");
2823                 } else
2824                         fprintf(stderr,"PROGRESS: Skipping Phase 2\n");
2825         }
2826         free(node_buffer.base);
2827         node_buffer.base=NULL;
2828         node_buffer.malloced=0;
2829         node_buffer.size=0;
2830         if (end == 2)
2831                 exit(0);
2832         } else {
2833                 ways_split=fopen("ways_split.tmp","wb+");
2834                 process_binfile(stdin, ways_split);
2835                 fclose(ways_split);
2836         }
2837         if (output == 1) {
2838                 fprintf(stderr,"PROGRESS: Phase 3: dumping\n");
2839                 if (process_nodes) {
2840                         nodes=fopen("nodes.tmp","rb");
2841                         if (nodes) {
2842                                 dump(nodes);
2843                                 fclose(nodes);
2844                         }
2845                 }
2846                 if (process_ways) {
2847                         ways_split=fopen("ways_split.tmp","rb");
2848                         if (ways_split) {
2849                                 dump(ways_split);
2850                                 fclose(ways_split);
2851                         }
2852                 }
2853                 exit(0);
2854         }
2855         if (start <= 3) {
2856                 phase=3;
2857                 fprintf(stderr,"PROGRESS: Phase 3: generating tiles\n");
2858                 if (process_ways)
2859                         ways_split=fopen("ways_split.tmp","rb");
2860                 if (process_nodes)
2861                         nodes=fopen("nodes.tmp","rb");
2862                 tilesdir=fopen("tilesdir.tmp","wb+");
2863                 phase3(ways_split,nodes,tilesdir);
2864                 fclose(tilesdir);
2865                 if (nodes)
2866                         fclose(nodes);
2867                 if (ways_split)
2868                         fclose(ways_split);
2869         }
2870         if (end == 3)
2871                 exit(0);
2872         if (start <= 4) {
2873                 phase=4;
2874                 fprintf(stderr,"PROGRESS: Phase 4: assembling map\n");
2875                 if (process_ways)
2876                         ways_split=fopen("ways_split.tmp","rb");
2877                 if (process_nodes)
2878                         nodes=fopen("nodes.tmp","rb");
2879                 res=fopen(result,"wb+");
2880                 zipdir=fopen("zipdir.tmp","wb+");
2881                 phase4(ways_split,nodes,res,zipdir,compression_level);
2882                 fclose(zipdir);
2883                 fclose(res);
2884                 if (nodes)
2885                         fclose(nodes);
2886                 if (ways_split)
2887                         fclose(ways_split);
2888                 if(!keep_tmpfiles) {
2889                         remove("nodes.tmp");
2890                         remove("ways_split.tmp");
2891                         remove("tilesdir.tmp");
2892                         remove("zipdir.tmp");
2893 #ifdef GENERATE_INDEX
2894                         remove_countryfiles();
2895 #endif
2896                 }
2897         }
2898         return 0;
2899 }