vcs.maemo.org Git - navit-package/blob - navit/osm2navit.c

   1 /**
   2  * Navit, a modular navigation system.
   3  * Copyright (C) 2005-2008 Navit Team
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License
   7  * version 2 as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write to the
  16  * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  17  * Boston, MA  02110-1301, USA.
  18  */
  19
  20
  21 #define _FILE_OFFSET_BITS 64
  22 #define _LARGEFILE_SOURCE
  23 #define _LARGEFILE64_SOURCE
  24 #include <glib.h>
  25 #include <assert.h>
  26 #include <string.h>
  27 #include <signal.h>
  28 #include <stdlib.h>
  29 #include <stdio.h>
  30 #include <math.h>
  31 #include <getopt.h>
  32 #include <unistd.h>
  33 #include <fcntl.h>
  34 #include <sys/stat.h>
  35 #include <zlib.h>
  36 #include "item.h"
  37 #include "zipfile.h"
  38 #include "config.h"
  39 #ifdef HAVE_POSTGRESQL
  40 #include <libpq-fe.h>
  41 #endif
  42
  43 #define BUFFER_SIZE 1280
  44
  45 #define GENERATE_INDEX
  46
  47 #if 1
  48 #define debug_tile(x) 0
  49 #else
  50 #define debug_tile(x) (!strcmp(x,"bcdbd") || !strcmp(x,"bcdbd") || !strcmp(x,"bcdbda") || !strcmp(x,"bcdbdb") || !strcmp(x,"bcdbdba") || !strcmp(x,"bcdbdbb") || !strcmp(x,"bcdbdbba") || !strcmp(x,"bcdbdbaa") || !strcmp(x,"bcdbdbacaa") || !strcmp(x,"bcdbdbacab") || !strcmp(x,"bcdbdbacaba") || !strcmp(x,"bcdbdbacabaa") || !strcmp(x,"bcdbdbacabab") || !strcmp(x,"bcdbdbacababb") || !strcmp(x,"bcdbdbacababba") || !strcmp(x,"bcdbdbacababbb") || !strcmp(x,"bcdbdbacababbd") || !strcmp(x,"bcdbdbacababaa") || !strcmp(x,"bcdbdbacababab") || !strcmp(x,"bcdbdbacababac") || !strcmp(x,"bcdbdbacababad") || !strcmp(x,"bcdbdbacabaaa") || !strcmp(x,"bcdbdbacabaaba") || !strcmp(x,"bcdbdbacabaabb") || !strcmp(x,"bcdbdbacabaabc") || !strcmp(x,"bcdbdbacabaabd") || !strcmp(x,"bcdbdbacabaaaa") || !strcmp(x,"bcdbdbacabaaab") || !strcmp(x,"bcdbdbacabaaac") || !strcmp(x,"bcdbdbacabaaad") || 0)
  51 #endif
  52
  53
  54 static GHashTable *dedupe_ways_hash;
  55
  56 static int attr_debug_level=1;
  57 static int nodeid,wayid;
  58 static int report,phase;
  59 static int ignore_unkown = 0, coverage=0;
  60
  61 static char *attrmap={
  62         "n      *=*                     point_unkn\n"
  63         "n      amenity=hospital        poi_hospital\n"
  64         "n      amenity=atm             poi_bank\n"
  65         "n      amenity=bank            poi_bank\n"
  66         "n      amenity=pub             poi_bar\n"
  67         "n      amenity=cafe            poi_cafe\n"
  68         "n      amenity=bus_station     poi_bus_station\n"
  69         "n      amenity=parking         poi_car_parking\n"
  70         "n      amenity=cinema          poi_cinema\n"
  71         "n      amenity=fire_station    poi_firebrigade\n"
  72         "n      amenity=fuel            poi_fuel\n"
  73         "n      amenity=courthouse      poi_justice\n"
  74         "n      amenity=library         poi_library\n"
  75         "n      amenity=pharmacy        poi_pharmacy\n"
  76         "n      amenity=place_of_worship,religion=christian     poi_church\n"
  77         "n      amenity=police          poi_police\n"
  78         "n      amenity=post_office     poi_post\n"
  79         "n      amenity=post_box        poi_post\n"
  80         "n      amenity=public_building poi_public_office\n"
  81         "n      amenity=restaurant      poi_restaurant\n"
  82         "n      amenity=fast_food       poi_fastfood\n"
  83         "n      amenity=toilets         poi_restroom\n"
  84         "n      amenity=school          poi_school\n"
  85         "n      amenity=university      poi_school\n"
  86         "n      amenity=college         poi_school\n"
  87         "n      amenity=telephone       poi_telephone\n"
  88         "n      amenity=theatre         poi_theater\n"
  89         "n      amenity=townhall                poi_townhall\n"
  90         "n      highway=bus_stop        poi_bus_stop\n"
  91         "n      highway=mini_roundabout mini_roundabout\n"
  92         "n      highway=motorway_junction       highway_exit\n"
  93         "n      highway=traffic_signals traffic_signals\n"
  94         "n      highway=turning_circle  turning_circle\n"
  95         "n      leisure=slipway         poi_boat_ramp\n"
  96         "n      leisure=fishing         poi_fish\n"
  97         "n      sport=golf      poi_golf\n"
  98         "n      leisure=golf_course     poi_golf\n"
  99         "n      leisure=marina          poi_marine\n"
 100         "n      leisure=sports_centre   poi_sport\n"
 101         "n      leisure=stadium         poi_stadium\n"
 102         "n      shop=supermarket        poi_shopping\n"
 103         "n      shop=convenience        poi_shop_grocery\n"
 104         "n      tourism=attraction      poi_attraction\n"
 105         "n      tourism=camp_site       poi_camp_rv\n"
 106         "n      tourism=caravan_site    poi_camp_rv\n"
 107         "n      tourism=hotel           poi_hotel\n"
 108         "n      tourism=motel           poi_hotel\n"
 109         "n      tourism=guest_house     poi_hotel\n"
 110         "n      tourism=hostel          poi_hotel\n"
 111         "n      tourism=information     poi_information\n"
 112         "n      tourism=museum          poi_museum_history\n"
 113         "n      tourism=picnic_site     poi_picnic\n"
 114         "n      tourism=theme_park      poi_resort\n"
 115         "n      tourism=zoo             poi_zoo\n"
 116         "n      amenity=grave_yard      poi_cemetery\n"
 117         "n      landuse=cemetery        poi_cemetery\n"
 118         "n      military=airfield       poi_military\n"
 119         "n      military=bunker         poi_military\n"
 120         "n      military=barracks       poi_military\n"
 121         "n      military=range          poi_military\n"
 122         "n      military=danger_area    poi_danger_area\n"
 123         "n      sport=swimming  poi_swimming\n"
 124         "n      sport=skiing            poi_skiing\n"
 125         "n      aeroway=aerodrome       poi_airport\n"
 126         "n      aeroway=airport         poi_airport\n"
 127         "n      aeroway=terminal        poi_airport\n"
 128         "n      aeroway=helipad         poi_heliport\n"
 129         "n      man_made=tower  poi_tower\n"
 130         "n      natural=bay             poi_bay\n"
 131         "n      natural=peak            poi_peak\n"
 132         "n      place=suburb            district_label\n"
 133         "n      place=city              town_label_2e5\n"
 134         "n      place=town              town_label_2e4\n"
 135         "n      place=village           town_label_2e3\n"
 136         "n      place=hamlet            town_label_2e2\n"
 137         "n      place=locality          town_label_2e0\n"
 138         "n      railway=halt            poi_rail_halt\n"
 139         "n      railway=level_crossing          poi_level_crossing\n"
 140         "n      railway=station         poi_rail_station\n"
 141         "n      railway=tram_stop               poi_rail_tram_stop\n"
 142         "n      amenity=park_bench              poi_bench\n"
 143         "n      amenity=bench           poi_bench\n"
 144         "n      amenity=biergarten              poi_biergarten\n"
 145         "n      historic=boundary_stone         poi_boundary_stone\n"
 146         "n      historic=castle         poi_castle\n"
 147         "n      historic=ruins          poi_ruins\n"
 148         "n      Annehmlichkeit=Hochsitz         poi_hunting_stand\n"
 149         "n      amenity=hunting_stand           poi_hunting_stand\n"
 150         "n      historic=memorial               poi_memorial\n"
 151         "n      historic=monument               poi_monument\n"
 152         "n      amenity=shelter         poi_shelter\n"
 153         "n      amenity=fountain                poi_fountain\n"
 154         "n      amenity=drinking_water          poi_potable_water\n"
 155         "n      amenity=toilets         poi_toilets\n"
 156         "n      tourism=viewpoint               poi_viewpoint\n"
 157         "w      *=*                     street_unkn\n"
 158         "w      contour_ext=elevation_major     height_line_1\n"
 159         "w      contour_ext=elevation_medium    height_line_2\n"
 160         "w      contour_ext=elevation_minor     height_line_3\n"
 161         "w      amenity=place_of_worship        poly_building\n"
 162         "w      building=*      poly_building\n"
 163         "w      aeroway=aerodrome       poly_airport\n"
 164         "w      aeroway=apron           poly_apron\n"
 165         "w      aeroway=runway          aeroway_runway\n"
 166         "w      aeroway=taxiway         aeroway_taxiway\n"
 167         "w      aeroway=terminal        poly_terminal\n"
 168         "w      highway=bridleway       bridleway\n"
 169         "w      highway=path,horse=designated   bridleway\n"
 170         "w      highway=cycleway        cycleway\n"
 171         "w      highway=cyclepath       cycleway\n"
 172         "w      highway=path,bicycle=designated cycleway\n"
 173         "w      highway=footway         footway\n"
 174         "w      piste:type=nordic       piste_nordic\n"
 175         "w      highway=footway,piste:type=nordic       footway_and_piste_nordic\n"
 176         "w      route=ski       piste_nordic\n"
 177         "w      piste:type=downhill,piste:difficulty=novice     piste_downhill_novice\n"
 178         "w      piste:type=downhill,piste:difficulty=easy       piste_downhill_easy\n"
 179         "w      piste:type=downhill,piste:difficulty=intermediate       piste_downhill_intermediate\n"
 180         "w      piste:type=downhill,piste:difficulty=advanced   piste_downhill_advanced\n"
 181         "w      piste:type=downhill,piste:difficulty=expert     piste_downhill_expert\n"
 182         "w      piste:type=downhill,piste:difficulty=freeride   piste_downhill_freeride\n"
 183         "w      highway=path,foot=designated    footway\n"
 184         "w      highway=steps   steps\n"
 185         "w      highway=path    path\n"
 186         "w      highway=path,sac_scale=hiking   hiking\n"
 187         "w      highway=path,sac_scale=mountain_hiking  hiking_mountain\n"
 188         "w      highway=path,sac_scale=demanding_mountain_hiking        hiking_mountain_demanding\n"
 189         "w      highway=path,sac_scale=alpine_hiking    hiking_alpine\n"
 190         "w      highway=path,sac_scale=demanding_alpine_hiking  hiking_alpine_demanding\n"
 191         "w      highway=path,sac_scale=difficult_alpine_hiking  hiking_alpine_difficult\n"
 192         "w      highway=unsurfaced      track_gravelled\n"
 193         "w      highway=track           track_gravelled\n"
 194         "w      highway=track,tracktype=grade1          track_paved\n"
 195         "w      highway=track,surface=paved             track_paved\n"
 196         "w      highway=track,tracktype=grade2          track_gravelled\n"
 197         "w      highway=track,surface=gravel            track_gravelled\n"
 198         "w      highway=track,tracktype=grade3          track_unpaved\n"
 199         "w      highway=track,surface=unpaved           track_unpaved\n"
 200         "w      highway=track,tracktype=grade4          track_ground\n"
 201         "w      highway=track,surface=ground            track_ground\n"
 202         "w      highway=track,tracktype=grade5          track_grass\n"
 203         "w      highway=track,surface=grass             track_grass\n"
 204         "w      highway=parking_lane    street_parking_lane\n"
 205         "w      highway=service,service=parking_aisle   street_parking_lane\n"
 206         "w      highway=service         street_service\n"
 207         "w      highway=service,area=1  poly_service\n"
 208         "w      highway=construction            street_construction\n"
 209         "w      highway=pedestrian      street_pedestrian\n"
 210         "w      highway=pedestrian,area=1       poly_pedestrian\n"
 211         "w      highway=plaza   poly_plaza\n"
 212         "w      landuse=plaza   poly_plaza\n"
 213         "w      highway=residential     street_1_city\n"
 214         "w      highway=residential,area=1      poly_street_1\n"
 215         "w      highway=living_street           living_street\n"
 216         "w      highway=unclassified    street_1_city\n"
 217         "w      highway=unclassified,area=1     poly_street_1\n"
 218         "w      highway=road    street_1_city\n"
 219         "w      highway=minor           street_1_land\n"
 220         "w      highway=tertiary        street_2_city\n"
 221         "w      highway=tertiary,area=1 poly_street_2\n"
 222         "w      highway=tertiary_link   ramp\n"
 223         "w      highway=secondary       street_3_city\n"
 224         "w      highway=secondary,area=1        poly_street_3\n"
 225         "w      highway=secondary_link  ramp\n"
 226         "w      highway=primary         street_4_city\n"
 227         "w      highway=primary_link    ramp\n"
 228         "w      highway=trunk           street_4_city\n"
 229         "w      highway=trunk_link      ramp\n"
 230         "w      highway=motorway        highway_city\n"
 231         "w      highway=motorway_link   ramp\n"
 232         "w      historic=town gate      poly_building\n"
 233         "w      cycleway=track          cycleway\n"
 234         "w      amenity=parking         poly_car_parking\n"
 235         "w      landuse=allotments      poly_allotments\n"
 236         "w      landuse=cemetery        poly_cemetery\n"
 237         "w      amenity=grave_yard      poly_cemetery\n"
 238         "w      landuse=forest          poly_wood\n"
 239         "w      landuse=industrial      poly_industry\n"
 240         "w      landuse=residential     poly_town\n"
 241         "w      landuse=residential,area=1      poly_town\n"
 242         "w      landuse=farmyard        poly_town\n"
 243         "w      amenity=college poly_college\n"
 244         "w      amenity=university      poly_university\n"
 245         "w      tourism=camp_site       poly_camp_site\n"
 246         "w      tourism=caravan_site    poly_caravan_site\n"
 247         "w      tourism=picnic_site     poly_picnic_site\n"
 248         "w      tourism=theme_park      poly_theme_park\n"
 249         "w      tourism=attraction      poly_attraction\n"
 250         "w      tourism=zoo     poly_zoo\n"
 251         "w      tourism=artwork poly_artwork\n"
 252         "w      historic=archaeological_site    poly_archaeological_site\n"
 253         "w      historic=ruins  poly_ruins\n"
 254         "w      historic=battlefield    poly_battlefield\n"
 255         "w      landuse=quarry  poly_quarry\n"
 256         "w      landuse=landfill        poly_landfill\n"
 257         "w      landuse=retail  poly_retail\n"
 258         "w      landuse=commercial      poly_commercial\n"
 259         "w      landuse=brownfield      poly_brownfield\n"
 260         "w      landuse=greenfield      poly_greenfield\n"
 261         "w      landuse=construction    poly_construction\n"
 262         "w      landuse=railway poly_railway\n"
 263         "w      natural=glacier poly_glacier\n"
 264         "w      natural=scree   poly_scree\n"
 265         "w      natural=scrub   poly_scrub\n"
 266         "w      natural=fell    poly_fell\n"
 267         "w      natural=heath   poly_heath\n"
 268         "w      natural=marsh   poly_marsh\n"
 269         "w      natural=mud     poly_mud\n"
 270         "w      natural=beach   poly_beach\n"
 271         "w      natural=land    poly_land\n"
 272         "w      landuse=basin   poly_basin\n"
 273         "w      landuse=reservoir       poly_reservoir\n"
 274         "w      landuse=farm    poly_farm\n"
 275         "w      landuse=farmland        poly_farm\n"
 276         "w      landuse=vineyard        poly_farm\n"
 277         "w      leisure=park            poly_park\n"
 278         "w      landuse=village_green           poly_village_green\n"
 279         "w      landuse=recreation_ground               poly_recreation_ground\n"
 280         "w      natural=wood            poly_wood\n"
 281         "w      natural=water           poly_water\n"
 282         "w      natural=coastline       water_line\n"
 283         "w      place=suburb            poly_town\n"
 284         "w      place=town              poly_town\n"
 285         "w      landuse=military        poly_military\n"
 286         "w      military=airfield       poly_airfield\n"
 287         "w      military=barracks       poly_barracks\n"
 288         "w      military=danger_area    poly_danger_area\n"
 289         "w      military=range  poly_range\n"
 290         "w      military=naval_base     poly_naval_base\n"
 291         "w      power=line      powerline\n"
 292         "w      railway=rail            rail\n"
 293         "w      railway=narrow_gauge    rail_narrow_gauge\n"
 294         "w      railway=light_rail              rail_light\n"
 295         "w      railway=subway          rail_subway\n"
 296         "w      railway=tram            rail_tram\n"
 297         "w      railway=monorail                rail_mono\n"
 298         "w      railway=preserved               rail_preserved\n"
 299         "w      railway=disused         rail_disused\n"
 300         "w      railway=abandoned               rail_abandoned\n"
 301         "w      aerialway=cable_car             lift_cable_car\n"
 302         "w      aerialway=chair_lift            lift_chair\n"
 303         "w      aerialway=drag_lift             lift_drag\n"
 304         "w      leisure=golf_course     poly_golf_course\n"
 305         "w      sport=*         poly_sport\n"
 306         "w      leisure=sports_centre           poly_sport\n"
 307         "w      leisure=stadium         poly_sports_stadium\n"
 308         "w      leisure=track           poly_sports_track\n"
 309         "w      leisure=pitch           poly_sports_pitch\n"
 310         "w      leisure=water_park              poly_water_park\n"
 311         "w      leisure=marina          poly_marina\n"
 312         "w      leisure=fishing         poly_fishing\n"
 313         "w      leisure=garden          poly_garden\n"
 314         "w      leisure=common          poly_common\n"
 315         "w      leisure=playground              poly_playground\n"
 316         "w      leisure=nature_reserve          poly_nature_reserve\n"
 317         "w      waterway=canal          water_canal\n"
 318         "w      waterway=river          water_river\n"
 319         "w      waterway=stream         water_stream\n"
 320         "w      waterway=drain          water_drain\n"
 321         "w      waterway=riverbank      poly_water\n"
 322         "w      boundary=administrative border_country\n"
 323         "w      boundary=civil  border_civil\n"
 324         "w      boundary=political      border_political\n"
 325         "w      boundary=national_park  border_national_park\n"
 326         "w      route=ferry             ferry\n"
 327         "w      highway=bus_guideway            bus_guideway\n"
 328 };
 329
 330 struct coord {
 331         int x;
 332         int y;
 333 } coord_buffer[65536];
 334
 335 #define IS_REF(c) ((c).x >= (1 << 30))
 336 #define REF(c) ((c).y)
 337 #define SET_REF(c,ref) do { (c).x = 1 << 30; (c).y = ref ; } while(0)
 338
 339 struct rect {
 340         struct coord l,h;
 341 };
 342
 343 static void bbox_extend(struct coord *c, struct rect *r);
 344
 345 #ifdef GENERATE_INDEX
 346
 347 static GHashTable *aux_tile_hash;
 348 GList *aux_tile_list;
 349
 350 struct country_table {
 351         int countryid;
 352         char *names;
 353         FILE *file;
 354         int size;
 355         int count;
 356         struct rect r;
 357 } country_table[] = {
 358         { 40,"Austria,�sterreich,AUT"},
 359         {124,"Canada"},
 360         {208,"Denmark,Danmark,DK"},
 361         {246,"Finland,Suomi"},
 362         {250,"France,R�publique fran�aise,FR"},
 363         {276,"Germany,Deutschland,Bundesrepublik Deutschland"},
 364         {380,"Italy,Italia"},
 365         {528,"Nederland,The Netherlands,Niederlande,NL"},
 366         {578,"Norway,Norge,Noreg,NO"},
 367         {724,"Spain,Espana,Espana,Reino de Espana"},
 368         {752,"Sweden,Sverige,Konungariket Sverige,SE"},
 369         {756,"Schweiz"},
 370         {826,"United Kingdom,UK"},
 371         {840,"USA"}
 372 };
 373
 374 static GHashTable *country_table_hash;
 375 #endif
 376
 377 struct attr_mapping {
 378         enum item_type type;
 379         int attr_present_idx_count;
 380         int attr_present_idx[0];
 381 };
 382
 383 static struct attr_mapping **attr_mapping_node;
 384 static int attr_mapping_node_count;
 385 static struct attr_mapping **attr_mapping_way;
 386 static int attr_mapping_way_count;
 387
 388 static char *attr_present;
 389 static int attr_present_count;
 390 static GHashTable *attr_hash;
 391
 392
 393 static GHashTable *strings_hash = NULL;
 394
 395
 396 static char* string_hash_lookup( const char* key )
 397 {
 398         char* key_ptr = NULL;
 399
 400         if ( strings_hash == NULL ) {
 401                 strings_hash = g_hash_table_new(g_str_hash, g_str_equal);
 402         }
 403
 404         if ( ( key_ptr = g_hash_table_lookup(strings_hash, key )) == NULL ) {
 405                 key_ptr = g_strdup( key );
 406                 g_hash_table_insert(strings_hash, key_ptr,  (gpointer)key_ptr );
 407
 408         }
 409         return key_ptr;
 410 }
 411
 412 static void
 413 build_attrmap_line(char *line)
 414 {
 415         char *t=NULL,*kvl=NULL,*i=NULL,*p,*kv;
 416         struct attr_mapping ***attr_mapping_curr,*attr_mapping=g_malloc0(sizeof(struct attr_mapping));
 417         int idx,attr_mapping_count=0,*attr_mapping_curr_count;
 418         t=line;
 419         p=strchr(t,'\t');
 420         if (p) {
 421                 while (*p == '\t')
 422                         *p++='\0';
 423                 kvl=p;
 424                 p=strchr(kvl,'\t');
 425         }
 426         if (p) {
 427                 while (*p == '\t')
 428                         *p++='\0';
 429                 i=p;
 430         }
 431         if (t[0] == 'w') {
 432                 if (! i)
 433                         i="street_unkn";
 434                 attr_mapping_curr=&attr_mapping_way;
 435                 attr_mapping_curr_count=&attr_mapping_way_count;
 436         } else {
 437                 if (! i)
 438                         i="point_unkn";
 439                 attr_mapping_curr=&attr_mapping_node;
 440                 attr_mapping_curr_count=&attr_mapping_node_count;
 441         }
 442         attr_mapping->type=item_from_name(i);
 443         while ((kv=strtok(kvl, ","))) {
 444                 kvl=NULL;
 445                 if (!(idx=(int)g_hash_table_lookup(attr_hash, kv))) {
 446                         idx=attr_present_count++;
 447                         g_hash_table_insert(attr_hash, kv, (gpointer) idx);
 448                 }
 449                 attr_mapping=g_realloc(attr_mapping, sizeof(struct attr_mapping)+(attr_mapping_count+1)*sizeof(int));
 450                 attr_mapping->attr_present_idx[attr_mapping_count++]=idx;
 451                 attr_mapping->attr_present_idx_count=attr_mapping_count;
 452         }
 453         *attr_mapping_curr=g_realloc(*attr_mapping_curr, sizeof(**attr_mapping_curr)*(*attr_mapping_curr_count+1));
 454         (*attr_mapping_curr)[(*attr_mapping_curr_count)++]=attr_mapping;
 455 }
 456
 457 static void
 458 build_attrmap(char *map)
 459 {
 460         char *p;
 461         attr_hash=g_hash_table_new(g_str_hash, g_str_equal);
 462         attr_present_count=1;
 463         while (map) {
 464                 p=strchr(map,'\n');
 465                 if (p)
 466                         *p++='\0';
 467                 if (strlen(map))
 468                         build_attrmap_line(map);
 469                 map=p;
 470         }
 471         attr_present=g_malloc0(sizeof(*attr_present)*attr_present_count);
 472 }
 473
 474 #ifdef GENERATE_INDEX
 475 static void
 476 build_countrytable(void)
 477 {
 478         int i;
 479         char *names,*str,*tok;
 480         country_table_hash=g_hash_table_new(g_str_hash, g_str_equal);
 481         for (i = 0 ; i < sizeof(country_table)/sizeof(struct country_table) ; i++) {
 482                 names=g_strdup(country_table[i].names);
 483                 str=names;
 484                 while ((tok=strtok(str, ","))) {
 485                         str=NULL;
 486                         g_hash_table_insert(country_table_hash, tok,  (gpointer)&country_table[i]);
 487                 }
 488         }
 489 }
 490 #endif
 491
 492
 493
 494 static int processed_nodes, processed_nodes_out, processed_ways, processed_relations, processed_tiles;
 495 static int in_way, in_node, in_relation;
 496
 497 static void
 498 sig_alrm(int sig)
 499 {
 500 #ifndef _WIN32
 501         signal(SIGALRM, sig_alrm);
 502 #endif
 503         alarm(30);
 504         fprintf(stderr,"PROGRESS%d: Processed %d nodes (%d out) %d ways %d relations %d tiles\n", phase, processed_nodes, processed_nodes_out, processed_ways, processed_relations, processed_tiles);
 505 }
 506
 507 struct item_bin {
 508         int len;
 509         enum item_type type;
 510         int clen;
 511 } item;
 512
 513 struct attr_bin {
 514         int len;
 515         enum attr_type type;
 516 };
 517
 518 struct attr_bin label_attr = {
 519         0, attr_label
 520 };
 521 char label_attr_buffer[BUFFER_SIZE];
 522
 523 struct attr_bin town_name_attr = {
 524         0, attr_town_name
 525 };
 526
 527 struct attr_bin street_name_attr = {
 528         0, attr_street_name
 529 };
 530
 531 struct attr_bin street_name_systematic_attr = {
 532         0, attr_street_name_systematic
 533 };
 534 char street_name_systematic_attr_buffer[BUFFER_SIZE];
 535
 536 struct attr_bin debug_attr = {
 537         0, attr_debug
 538 };
 539 char debug_attr_buffer[BUFFER_SIZE];
 540
 541 struct attr_bin flags_attr = {
 542         0, attr_flags
 543 };
 544 int flags_attr_value;
 545
 546 char is_in_buffer[BUFFER_SIZE];
 547
 548
 549 static void write_zipmember(FILE *out, FILE *dir_out, char *name, int filelen, char *data, int data_size, int compression_level);
 550
 551 static void
 552 pad_text_attr(struct attr_bin *a, char *buffer)
 553 {
 554         int l;
 555         if (buffer && buffer[0]) {
 556                 l=strlen(buffer)+1;
 557                 while (l % 4)
 558                         buffer[l++]='\0';
 559                 a->len=l/4+1;
 560         } else
 561                 a->len=0;
 562 }
 563
 564 static int
 565 xml_get_attribute(char *xml, char *attribute, char *buffer, int buffer_size)
 566 {
 567         int len=strlen(attribute);
 568         char *pos,*i,s,attr[len+2];
 569         strcpy(attr, attribute);
 570         strcpy(attr+len, "=");
 571         pos=strstr(xml, attr);
 572         if (! pos)
 573                 return 0;
 574         pos+=len+1;
 575         s=*pos++;
 576         if (! s)
 577                 return 0;
 578         i=strchr(pos, s);
 579         if (! i)
 580                 return 0;
 581         if (i - pos > buffer_size) {
 582                 fprintf(stderr,"Buffer overflow %d vs %d\n", i-pos, buffer_size);
 583                 return 0;
 584         }
 585         strncpy(buffer, pos, i-pos);
 586         buffer[i-pos]='\0';
 587         return 1;
 588 }
 589
 590 static int node_is_tagged;
 591
 592 static void
 593 add_tag(char *k, char *v)
 594 {
 595         GHashTable *value_hash;
 596         enum item_type type;
 597         int idx,level=2;
 598         char buffer[BUFFER_SIZE*2+2];
 599         if (! strcmp(k,"ele"))
 600                 level=9;
 601         if (! strcmp(k,"time"))
 602                 level=9;
 603         if (! strcmp(k,"created_by"))
 604                 level=9;
 605         if (! strncmp(k,"tiger:",6) || !strcmp(k,"AND_nodes"))
 606                 level=9;
 607         if (! strcmp(k,"converted_by") || ! strcmp(k,"source"))
 608                 level=8;
 609         if (! strncmp(k,"osmarender:",11) || !strncmp(k,"svg:",4))
 610                 level=8;
 611         if (! strcmp(k,"layer"))
 612                 level=7;
 613         if (! strcasecmp(v,"true") || ! strcasecmp(v,"yes"))
 614                 v="1";
 615         if (! strcmp(k,"oneway")) {
 616                 if (!strcmp(v,"1")) {
 617                         flags_attr_value=AF_ONEWAY;
 618                         flags_attr.len=2;
 619                 }
 620                 if (! strcmp(v,"-1")) {
 621                         flags_attr_value=AF_ONEWAYREV;
 622                         flags_attr.len=2;
 623                 }
 624                 if (!in_way)
 625                         level=6;
 626                 else
 627                         level=5;
 628         }
 629         if (! strcmp(k,"junction")) {
 630                 if (! strcmp(v,"roundabout")) {
 631                         flags_attr_value=AF_ONEWAY;
 632                         flags_attr.len=2;
 633                 }
 634         }
 635         if (! strcmp(k,"maxspeed")) {
 636                 level=5;
 637         }
 638         if (! strcmp(k,"bicycle")) {
 639                 level=5;
 640         }
 641         if (! strcmp(k,"foot")) {
 642                 level=5;
 643         }
 644         if (! strcmp(k,"note"))
 645                 level=5;
 646         if (! strcmp(k,"name")) {
 647                 strcpy(label_attr_buffer, v);
 648                 pad_text_attr(&label_attr, label_attr_buffer);
 649                 level=5;
 650         }
 651         if (! strcmp(k,"ref")) {
 652                 if (in_way) {
 653                         strcpy(street_name_systematic_attr_buffer, v);
 654                         pad_text_attr(&street_name_systematic_attr, street_name_systematic_attr_buffer);
 655                 }
 656                 level=5;
 657         }
 658         if (! strcmp(k,"is_in")) {
 659                 strcpy(is_in_buffer, v);
 660                 level=5;
 661         }
 662         if (! strcmp(k,"gnis:ST_alpha")) {
 663                 /*      assume a gnis tag means it is part of the USA:
 664                         http://en.wikipedia.org/wiki/Geographic_Names_Information_System
 665                         many US towns do not have is_in tags
 666                 */
 667                 strcpy(is_in_buffer, "USA");
 668                 level=5;
 669         }
 670         if (! strcmp(k,"lanes")) {
 671                 level=5;
 672         }
 673         if (attr_debug_level >= level) {
 674                 int bytes_left = sizeof( debug_attr_buffer ) - strlen(debug_attr_buffer) - 1;
 675                 if ( bytes_left > 0 )
 676                 {
 677                         snprintf(debug_attr_buffer+strlen(debug_attr_buffer), bytes_left,  " %s=%s", k, v);
 678                         debug_attr_buffer[ sizeof( debug_attr_buffer ) -  1 ] = '\0';
 679                         node_is_tagged=1;
 680                 }
 681         }
 682         if (level < 6)
 683                 node_is_tagged=1;
 684         if (level >= 5)
 685                 return;
 686
 687         strcpy(buffer,"*=*");
 688         if ((idx=(int)g_hash_table_lookup(attr_hash, buffer)))
 689                 attr_present[idx]=1;
 690
 691         sprintf(buffer,"%s=*", k);
 692         if ((idx=(int)g_hash_table_lookup(attr_hash, buffer)))
 693                 attr_present[idx]=2;
 694
 695         sprintf(buffer,"*=%s", v);
 696         if ((idx=(int)g_hash_table_lookup(attr_hash, buffer)))
 697                 attr_present[idx]=2;
 698
 699         sprintf(buffer,"%s=%s", k, v);
 700         if ((idx=(int)g_hash_table_lookup(attr_hash, buffer)))
 701                 attr_present[idx]=4;
 702 }
 703
 704 struct entity {
 705         char *entity;
 706         char c;
 707 } entities[]= {
 708         {"&quot;",'"'},
 709         {"&apos;",'\''},
 710         {"&amp;",'&'},
 711         {"&lt;",'<'},
 712         {"&gt;",'>'},
 713 };
 714
 715 static void
 716 decode_entities(char *buffer)
 717 {
 718         char *pos=buffer;
 719         int i,len,found;
 720
 721         while ((pos=strchr(pos, '&'))) {
 722                 found=0;
 723                 for (i = 0 ; i < sizeof(entities)/sizeof(struct entity); i++) {
 724                         len=strlen(entities[i].entity);
 725                         if (!strncmp(pos, entities[i].entity, len)) {
 726                                 *pos=entities[i].c;
 727                                 memmove(pos+1, pos+len, strlen(pos+len)+1);
 728                                 found=1;
 729                                 break;
 730                         }
 731                 }
 732                 pos++;
 733         }
 734 }
 735
 736 static int
 737 parse_tag(char *p)
 738 {
 739         char k_buffer[BUFFER_SIZE];
 740         char v_buffer[BUFFER_SIZE];
 741         if (!xml_get_attribute(p, "k", k_buffer, BUFFER_SIZE))
 742                 return 0;
 743         if (!xml_get_attribute(p, "v", v_buffer, BUFFER_SIZE))
 744                 return 0;
 745         decode_entities(v_buffer);
 746         add_tag(k_buffer, v_buffer);
 747         return 1;
 748 }
 749
 750
 751 struct buffer {
 752         int malloced_step;
 753         size_t malloced;
 754         unsigned char *base;
 755         size_t size;
 756 };
 757
 758 static struct tile_head {
 759         int num_subtiles;
 760         int total_size;
 761         char *name;
 762         char *zip_data;
 763         int total_size_used;
 764         int zipnum;
 765         int process;
 766         struct tile_head *next;
 767         // char subtiles[0];
 768 } *tile_head_root;
 769
 770
 771 int coord_count;
 772
 773 struct node_item {
 774         int id;
 775         char ref_node;
 776         char ref_way;
 777         char ref_ref;
 778         char dummy;
 779         struct coord c;
 780 };
 781
 782 static struct buffer node_buffer = {
 783         64*1024*1024,
 784 };
 785
 786
 787 static char** th_get_subtile( const struct tile_head* th, int idx )
 788 {
 789         char* subtile_ptr = NULL;
 790         subtile_ptr = (char*)th + sizeof( struct tile_head ) + idx * sizeof( char *);
 791         return (char**)subtile_ptr;
 792 }
 793
 794 static void
 795 extend_buffer(struct buffer *b)
 796 {
 797         b->malloced+=b->malloced_step;
 798         b->base=realloc(b->base, b->malloced);
 799         if (b->base == NULL) {
 800                 fprintf(stderr,"realloc of %d bytes failed\n",b->malloced);
 801                 exit(1);
 802         }
 803
 804 }
 805
 806 int nodeid_last;
 807 GHashTable *node_hash;
 808
 809 static void
 810 node_buffer_to_hash(void)
 811 {
 812         int i,count=node_buffer.size/sizeof(struct node_item);
 813         struct node_item *ni=(struct node_item *)node_buffer.base;
 814         for (i = 0 ; i < count ; i++)
 815                 g_hash_table_insert(node_hash, (gpointer)(ni[i].id), (gpointer)i);
 816 }
 817
 818 static struct node_item *ni;
 819
 820 static void
 821 add_node(int id, double lat, double lon)
 822 {
 823         if (node_buffer.size + sizeof(struct node_item) > node_buffer.malloced)
 824                 extend_buffer(&node_buffer);
 825         node_is_tagged=0;
 826         nodeid=id;
 827         item.type=type_point_unkn;
 828         label_attr.len=0;
 829         town_name_attr.len=0;
 830         debug_attr.len=0;
 831         is_in_buffer[0]='\0';
 832         sprintf(debug_attr_buffer,"nodeid=%d", nodeid);
 833         ni=(struct node_item *)(node_buffer.base+node_buffer.size);
 834         ni->id=id;
 835         ni->ref_node=0;
 836         ni->ref_way=0;
 837         ni->ref_ref=0;
 838         ni->dummy=0;
 839         ni->c.x=lon*6371000.0*M_PI/180;
 840         ni->c.y=log(tan(M_PI_4+lat*M_PI/360))*6371000.0;
 841         node_buffer.size+=sizeof(struct node_item);
 842         if (! node_hash) {
 843                 if (ni->id > nodeid_last) {
 844                         nodeid_last=ni->id;
 845                 } else {
 846                         fprintf(stderr,"INFO: Nodes out of sequence (new %d vs old %d), adding hash\n", ni->id, nodeid_last);
 847                         node_hash=g_hash_table_new(NULL, NULL);
 848                         node_buffer_to_hash();
 849                 }
 850         } else
 851                 if (!g_hash_table_lookup(node_hash, (gpointer)(ni->id)))
 852                         g_hash_table_insert(node_hash, (gpointer)(ni->id), (gpointer)(ni-(struct node_item *)node_buffer.base));
 853                 else {
 854                         node_buffer.size-=sizeof(struct node_item);
 855                         nodeid=0;
 856                 }
 857
 858 }
 859
 860 static int
 861 parse_node(char *p)
 862 {
 863         char id_buffer[BUFFER_SIZE];
 864         char lat_buffer[BUFFER_SIZE];
 865         char lon_buffer[BUFFER_SIZE];
 866         if (!xml_get_attribute(p, "id", id_buffer, BUFFER_SIZE))
 867                 return 0;
 868         if (!xml_get_attribute(p, "lat", lat_buffer, BUFFER_SIZE))
 869                 return 0;
 870         if (!xml_get_attribute(p, "lon", lon_buffer, BUFFER_SIZE))
 871                 return 0;
 872         add_node(atoi(id_buffer), atof(lat_buffer), atof(lon_buffer));
 873         return 1;
 874 }
 875
 876
 877 static struct node_item *
 878 node_item_get(int id)
 879 {
 880         struct node_item *ni=(struct node_item *)(node_buffer.base);
 881         int count=node_buffer.size/sizeof(struct node_item);
 882         int interval=count/4;
 883         int p=count/2;
 884         if (node_hash) {
 885                 int i;
 886                 i=(int)(g_hash_table_lookup(node_hash, (gpointer)id));
 887                 return ni+i;
 888         }
 889         while (ni[p].id != id) {
 890 #if 0
 891                 fprintf(stderr,"p=%d count=%d interval=%d id=%d ni[p].id=%d\n", p, count, interval, id, ni[p].id);
 892 #endif
 893                 if (ni[p].id < id) {
 894                         p+=interval;
 895                         if (interval == 1) {
 896                                 if (p >= count)
 897                                         return NULL;
 898                                 if (ni[p].id > id)
 899                                         return NULL;
 900                         } else {
 901                                 if (p >= count)
 902                                         p=count-1;
 903                         }
 904                 } else {
 905                         p-=interval;
 906                         if (interval == 1) {
 907                                 if (p < 0)
 908                                         return NULL;
 909                                 if (ni[p].id < id)
 910                                         return NULL;
 911                         } else {
 912                                 if (p < 0)
 913                                         p=0;
 914                         }
 915                 }
 916                 if (interval > 1)
 917                         interval/=2;
 918         }
 919
 920         return &ni[p];
 921 }
 922
 923 static void
 924 node_ref_way(int id)
 925 {
 926         struct node_item *ni=node_item_get(id);
 927         if (! ni) {
 928                 fprintf(stderr,"WARNING: node id %d not found\n", id);
 929                 return;
 930         }
 931         ni->ref_way++;
 932 }
 933
 934
 935 static void
 936 add_way(int id)
 937 {
 938         wayid=id;
 939         coord_count=0;
 940         item.type=type_street_unkn;
 941         label_attr.len=0;
 942         street_name_attr.len=0;
 943         street_name_systematic_attr.len=0;
 944         debug_attr.len=0;
 945         flags_attr.len=0;
 946         sprintf(debug_attr_buffer,"wayid=%d", wayid);
 947 }
 948
 949 static int
 950 parse_way(char *p)
 951 {
 952         char id_buffer[BUFFER_SIZE];
 953         if (!xml_get_attribute(p, "id", id_buffer, BUFFER_SIZE))
 954                 return 0;
 955         add_way(atoi(id_buffer));
 956         return 1;
 957 }
 958
 959 static int
 960 parse_relation(char *p)
 961 {
 962         debug_attr_buffer[0]='\0';
 963         return 1;
 964 }
 965
 966 static void
 967 item_buffer_set_type(char *buffer, enum item_type type)
 968 {
 969         struct item_bin *ib=(struct item_bin *) buffer;
 970         ib->clen=0;
 971         ib->len=2;
 972         ib->type=type;
 973 }
 974
 975 static void
 976 item_buffer_add_coord(char *buffer, struct coord *c, int count)
 977 {
 978         struct item_bin *ib=(struct item_bin *) buffer;
 979         struct coord *c2=(struct coord *)(ib+1);
 980         c2+=ib->clen/2;
 981         memcpy(c2, c, count*sizeof(struct coord));
 982         ib->clen+=count*2;
 983         ib->len+=count*2;
 984 }
 985
 986 static void
 987 item_buffer_add_attr(char *buffer, struct attr *attr)
 988 {
 989         struct item_bin *ib=(struct item_bin *) buffer;
 990         struct attr_bin *ab=(struct attr_bin *)((int *)ib+ib->len+1);
 991         if (attr->type >= attr_type_string_begin && attr->type <= attr_type_string_end) {
 992                 ab->type=attr->type;
 993                 strcpy((char *)(ab+1),attr->u.str);
 994                 pad_text_attr(ab, (char *)(ab+1));
 995                 ib->len+=ab->len+1;
 996         }
 997 }
 998
 999 static void
1000 item_buffer_write(char *buffer, FILE *out)
1001 {
1002         struct item_bin *ib=(struct item_bin *) buffer;
1003         fwrite(buffer, (ib->len+1)*4, 1, out);
1004 }
1005
1006 static void
1007 write_attr(FILE *out, struct attr_bin *attr, void *buffer)
1008 {
1009         if (attr->len) {
1010                 fwrite(attr, sizeof(*attr), 1, out);
1011                 fwrite(buffer, (attr->len-1)*4, 1, out);
1012         }
1013 }
1014
1015 static int
1016 attr_longest_match(struct attr_mapping **mapping, int mapping_count, enum item_type *types, int types_count)
1017 {
1018         int i,j,longest=0,ret=0,sum,val;
1019         struct attr_mapping *curr;
1020         for (i = 0 ; i < mapping_count ; i++) {
1021                 sum=0;
1022                 curr=mapping[i];
1023                 for (j = 0 ; j < curr->attr_present_idx_count ; j++) {
1024                         val=attr_present[curr->attr_present_idx[j]];
1025                         if (val)
1026                                 sum+=val;
1027                         else {
1028                                 sum=-1;
1029                                 break;
1030                         }
1031                 }
1032                 if (sum > longest) {
1033                         longest=sum;
1034                         ret=0;
1035                 }
1036                 if (sum > 0 && sum == longest && ret < types_count)
1037                         types[ret++]=curr->type;
1038         }
1039         memset(attr_present, 0, sizeof(*attr_present)*attr_present_count);
1040         return ret;
1041 }
1042
1043 static void
1044 end_way(FILE *out)
1045 {
1046         int alen=0,count;
1047         enum item_type types[5];
1048
1049         if (! out)
1050                 return;
1051         if (dedupe_ways_hash) {
1052                 if (g_hash_table_lookup(dedupe_ways_hash, (gpointer)wayid))
1053                         return;
1054                 g_hash_table_insert(dedupe_ways_hash, (gpointer)wayid, (gpointer)1);
1055         }
1056         count=attr_longest_match(attr_mapping_way, attr_mapping_way_count, types, sizeof(types)/sizeof(enum item_type));
1057         pad_text_attr(&debug_attr, debug_attr_buffer);
1058         if (label_attr.len)
1059                 alen+=label_attr.len+1;
1060         if (street_name_systematic_attr.len)
1061                 alen+=street_name_systematic_attr.len+1;
1062         if (debug_attr.len)
1063                 alen+=debug_attr.len+1;
1064         if (flags_attr.len)
1065                 alen+=flags_attr.len+1;
1066         if (count)
1067                 item.type=types[0];
1068         else
1069                 item.type=type_street_unkn;
1070         if (coverage && item_is_street(item))
1071                 item.type=type_coverage;
1072         item.clen=coord_count*2;
1073         item.len=item.clen+2+alen;
1074         fwrite(&item, sizeof(item), 1, out);
1075         fwrite(coord_buffer, coord_count*sizeof(struct coord), 1, out);
1076         if (item_is_street(item)) {
1077                 street_name_attr.len=label_attr.len;
1078                 write_attr(out, &street_name_attr, label_attr_buffer);
1079         } else
1080                 write_attr(out, &label_attr, label_attr_buffer);
1081         write_attr(out, &street_name_systematic_attr, street_name_systematic_attr_buffer);
1082         write_attr(out, &debug_attr, debug_attr_buffer);
1083         write_attr(out, &flags_attr, &flags_attr_value);
1084 }
1085
1086 static void
1087 end_node(FILE *out)
1088 {
1089         int alen=0,conflict=0,count;
1090         enum item_type types[5];
1091         struct country_table *result=NULL, *lookup;
1092         if (!out || ! node_is_tagged || ! nodeid)
1093                 return;
1094         count=attr_longest_match(attr_mapping_node, attr_mapping_node_count, types, sizeof(types)/sizeof(enum item_type));
1095         pad_text_attr(&debug_attr, debug_attr_buffer);
1096         if (label_attr.len)
1097                 alen+=label_attr.len+1;
1098         if (debug_attr.len)
1099                 alen+=debug_attr.len+1;
1100         if (count)
1101                 item.type=types[0];
1102         else
1103                 item.type=type_point_unkn;
1104         item.clen=2;
1105         item.len=item.clen+2+alen;
1106         fwrite(&item, sizeof(item), 1, out);
1107         fwrite(&ni->c, 1*sizeof(struct coord), 1, out);
1108         if (item_is_town(item)) {
1109                 town_name_attr.len=label_attr.len;
1110                 write_attr(out, &town_name_attr, label_attr_buffer);
1111         } else
1112                 write_attr(out, &label_attr, label_attr_buffer);
1113         write_attr(out, &debug_attr, debug_attr_buffer);
1114 #ifdef GENERATE_INDEX
1115         if (item_is_town(item) && town_name_attr.len) {
1116                 char *tok,*buf=is_in_buffer;
1117                 while ((tok=strtok(buf, ","))) {
1118                         while (*tok==' ')
1119                                 tok++;
1120                         lookup=g_hash_table_lookup(country_table_hash,tok);
1121                         if (lookup) {
1122                                 if (result && result->countryid != lookup->countryid) {
1123                                         fprintf(stderr,"conflict for %s %s country %d vs %d\n", label_attr_buffer, debug_attr_buffer, lookup->countryid, result->countryid);
1124                                         conflict=1;
1125                                 } else
1126                                         result=lookup;
1127                         }
1128                         buf=NULL;
1129                 }
1130                 if (result && !conflict) {
1131                         if (!result->file) {
1132                                 char *name=g_strdup_printf("country_%d.bin.unsorted", result->countryid);
1133                                 result->file=fopen(name,"w");
1134                                 g_free(name);
1135                         }
1136                         if (result->file) {
1137                                 item.clen=2;
1138                                 item.len=item.clen+2+label_attr.len+1;
1139                                 fwrite(&item, sizeof(item), 1, result->file);
1140                                 fwrite(&ni->c, 1*sizeof(struct coord), 1, result->file);
1141                                 write_attr(result->file, &town_name_attr, label_attr_buffer);
1142                                 result->count++;
1143                                 result->size+=(item.clen+3+label_attr.len+1)*4;
1144                         }
1145
1146                 }
1147         }
1148 #endif
1149         processed_nodes_out++;
1150 }
1151
1152 static int
1153 sort_countries_compare(const void *p1, const void *p2)
1154 {
1155         struct item_bin *ib1=*((struct item_bin **)p1),*ib2=*((struct item_bin **)p2);
1156         struct attr_bin *attr1,*attr2;
1157         char *s1,*s2;
1158         assert(ib1->clen==2);
1159         assert(ib2->clen==2);
1160         attr1=(struct attr_bin *)((int *)(ib1+1)+ib1->clen);
1161         attr2=(struct attr_bin *)((int *)(ib2+1)+ib1->clen);
1162         assert(attr1->type == attr_town_name);
1163         assert(attr2->type == attr_town_name);
1164         s1=(char *)(attr1+1);
1165         s2=(char *)(attr2+1);
1166         return strcmp(s1, s2);
1167 #if 0
1168         fprintf(stderr,"sort_countries_compare p1=%p p2=%p %s %s\n",p1,p2,s1,s2);
1169 #endif
1170         return 0;
1171 }
1172
1173 #ifdef GENERATE_INDEX
1174 static void
1175 sort_countries(void)
1176 {
1177         int i,j;
1178         struct country_table *co;
1179         struct coord *c;
1180         struct item_bin *ib;
1181         FILE *f;
1182         char *p,*buffer,**idx,*name;
1183         for (i = 0 ; i < sizeof(country_table)/sizeof(struct country_table) ; i++) {
1184                 co=&country_table[i];
1185                 if (co->file) {
1186                         fclose(co->file);
1187                         co->file=NULL;
1188                 }
1189                 if (co->size) {
1190                         buffer=malloc(co->size);
1191                         assert(buffer != NULL);
1192                         idx=malloc(co->count*sizeof(void *));
1193                         assert(idx != NULL);
1194                         name=g_strdup_printf("country_%d.bin.unsorted", co->countryid);
1195                         f=fopen(name,"r");
1196                         assert(f != NULL);
1197                         fread(buffer, co->size, 1, f);
1198                         fclose(f);
1199                         unlink(name);
1200                         g_free(name);
1201                         p=buffer;
1202                         for (j = 0 ; j < co->count ; j++) {
1203                                 idx[j]=p;
1204                                 p+=(*((int *)p)+1)*4;
1205                         }
1206                         qsort(idx, co->count, sizeof(void *), sort_countries_compare);
1207                         name=g_strdup_printf("country_%d.bin", co->countryid);
1208                         f=fopen(name,"w");
1209                         for (j = 0 ; j < co->count ; j++) {
1210                                 ib=(struct item_bin *)(idx[j]);
1211                                 c=(struct coord *)(ib+1);
1212                                 fwrite(ib, (ib->len+1)*4, 1, f);
1213                                 if (j)
1214                                         bbox_extend(c, &co->r);
1215                                 else
1216                                         co->r.l=*c;
1217                                         co->r.h=*c;
1218                         }
1219                         fclose(f);
1220                 }
1221         }
1222 }
1223 #endif
1224
1225 static void
1226 add_nd(char *p, int ref)
1227 {
1228         int len;
1229         struct node_item *ni;
1230         ni=node_item_get(ref);
1231         if (ni) {
1232 #if 0
1233                 coord_buffer[coord_count++]=ni->c;
1234 #else
1235                 SET_REF(coord_buffer[coord_count], ref);
1236                 coord_count++;
1237 #endif
1238                 ni->ref_way++;
1239         } else {
1240                 len=strlen(p);
1241                 if (len > 0 && p[len-1]=='\n')
1242                         p[len-1]='\0';
1243                 fprintf(stderr,"WARNING: way %d: node %d not found (%s)\n",wayid,ref,p);
1244         }
1245         if (coord_count > 65536) {
1246                 fprintf(stderr,"ERROR: Overflow\n");
1247                 exit(1);
1248         }
1249 }
1250
1251 static int
1252 parse_nd(char *p)
1253 {
1254         char ref_buffer[BUFFER_SIZE];
1255         if (!xml_get_attribute(p, "ref", ref_buffer, BUFFER_SIZE))
1256                 return 0;
1257         add_nd(p, atoi(ref_buffer));
1258         return 1;
1259 }
1260
1261
1262 static void
1263 save_buffer(char *filename, struct buffer *b)
1264 {
1265         FILE *f;
1266         f=fopen(filename,"wb+");
1267         fwrite(b->base, b->size, 1, f);
1268         fclose(f);
1269 }
1270
1271 static void
1272 load_buffer(char *filename, struct buffer *b)
1273 {
1274         FILE *f;
1275         if (b->base)
1276                 free(b->base);
1277         b->malloced=0;
1278         f=fopen(filename,"rb");
1279         fseek(f, 0, SEEK_END);
1280         b->size=b->malloced=ftell(f);
1281         fprintf(stderr,"reading %d bytes from %s\n", b->size, filename);
1282         fseek(f, 0, SEEK_SET);
1283         b->base=malloc(b->size);
1284         assert(b->base != NULL);
1285         fread(b->base, b->size, 1, f);
1286         fclose(f);
1287 }
1288
1289 static int
1290 phase1(FILE *in, FILE *out_ways, FILE *out_nodes)
1291 {
1292         int size=4096;
1293         char buffer[size];
1294         char *p;
1295         sig_alrm(0);
1296         while (fgets(buffer, size, in)) {
1297                 p=strchr(buffer,'<');
1298                 if (! p) {
1299                         fprintf(stderr,"WARNING: wrong line %s\n", buffer);
1300                         continue;
1301                 }
1302                 if (!strncmp(p, "<?xml ",6)) {
1303                 } else if (!strncmp(p, "<osm ",5)) {
1304                 } else if (!strncmp(p, "<bound ",7)) {
1305                 } else if (!strncmp(p, "<node ",6)) {
1306                         if (!parse_node(p))
1307                                 fprintf(stderr,"WARNING: failed to parse %s\n", buffer);
1308                         in_node=1;
1309                         processed_nodes++;
1310                 } else if (!strncmp(p, "<tag ",5)) {
1311                         if (!parse_tag(p))
1312                                 fprintf(stderr,"WARNING: failed to parse %s\n", buffer);
1313                 } else if (!strncmp(p, "<way ",5)) {
1314                         in_way=1;
1315                         if (!parse_way(p))
1316                                 fprintf(stderr,"WARNING: failed to parse %s\n", buffer);
1317                         processed_ways++;
1318                 } else if (!strncmp(p, "<nd ",4)) {
1319                         if (!parse_nd(p))
1320                                 fprintf(stderr,"WARNING: failed to parse %s\n", buffer);
1321                 } else if (!strncmp(p, "<relation ",10)) {
1322                         in_relation=1;
1323                         if (!parse_relation(p))
1324                                 fprintf(stderr,"WARNING: failed to parse %s\n", buffer);
1325                         processed_relations++;
1326                 } else if (!strncmp(p, "<member ",8)) {
1327                 } else if (!strncmp(p, "</node>",7)) {
1328                         in_node=0;
1329                         end_node(out_nodes);
1330                 } else if (!strncmp(p, "</way>",6)) {
1331                         in_way=0;
1332                         end_way(out_ways);
1333                 } else if (!strncmp(p, "</relation>",11)) {
1334                         in_relation=0;
1335                 } else if (!strncmp(p, "</osm>",6)) {
1336                 } else {
1337                         fprintf(stderr,"WARNING: unknown tag in %s\n", buffer);
1338                 }
1339         }
1340         sig_alrm(0);
1341         alarm(0);
1342         return 1;
1343 }
1344
1345 #ifdef HAVE_POSTGRESQL
1346 static int
1347 phase1_db(char *dbstr, FILE *out_ways, FILE *out_nodes)
1348 {
1349         PGconn *conn;
1350         PGresult *res,*node,*way,*tag;
1351         int count,tagged,i,j,k;
1352         long min, max, id, tag_id, node_id;
1353         char query[256];
1354
1355         sig_alrm(0);
1356         conn=PQconnectdb(dbstr);
1357         if (! conn) {
1358                 fprintf(stderr,"Failed to connect to database with '%s'\n",dbstr);
1359                 exit(1);
1360         }
1361         res=PQexec(conn, "begin");
1362         if (! res) {
1363                 fprintf(stderr, "Cannot begin transaction: %s\n", PQerrorMessage(conn));
1364                 PQclear(res);
1365                 exit(1);
1366         }
1367         res=PQexec(conn, "set transaction isolation level serializable");
1368         if (! res) {
1369                 fprintf(stderr, "Cannot set isolation level: %s\n", PQerrorMessage(conn));
1370                 PQclear(res);
1371                 exit(1);
1372         }
1373         res=PQexec(conn, "declare node cursor for select id,x(coordinate),y(coordinate) from node order by id");
1374         if (! res) {
1375                 fprintf(stderr, "Cannot setup cursor for nodes: %s\n", PQerrorMessage(conn));
1376                 PQclear(res);
1377                 exit(1);
1378         }
1379         res=PQexec(conn, "declare way cursor for select id from way order by id");
1380         if (! res) {
1381                 fprintf(stderr, "Cannot setup cursor for nodes: %s\n", PQerrorMessage(conn));
1382                 PQclear(res);
1383                 exit(1);
1384         }
1385         res=PQexec(conn, "declare relation cursor for select id from relation order by id");
1386         if (! res) {
1387                 fprintf(stderr, "Cannot setup cursor for nodes: %s\n", PQerrorMessage(conn));
1388                 PQclear(res);
1389                 exit(1);
1390         }
1391         for (;;) {
1392                 node=PQexec(conn, "fetch 100000 from node");
1393                 if (! node) {
1394                         fprintf(stderr, "Cannot setup cursor for nodes: %s\n", PQerrorMessage(conn));
1395                         PQclear(node);
1396                         exit(1);
1397                 }
1398                 count=PQntuples(node);
1399                 if (! count)
1400                         break;
1401                 min=atol(PQgetvalue(node, 0, 0));
1402                 max=atol(PQgetvalue(node, count-1, 0));
1403                 sprintf(query,"select node_id,name,value from node_tag where node_id >= %ld and node_id <= %ld order by node_id", min, max);
1404                 tag=PQexec(conn, query);
1405                 if (! tag) {
1406                         fprintf(stderr, "Cannot query node_tag: %s\n", PQerrorMessage(conn));
1407                         exit(1);
1408                 }
1409                 j=0;
1410                 for (i = 0 ; i < count ; i++) {
1411                         id=atol(PQgetvalue(node, i, 0));
1412                         add_node(id, atof(PQgetvalue(node, i, 1)), atof(PQgetvalue(node, i, 2)));
1413                         tagged=0;
1414                         in_node=1;
1415                         processed_nodes++;
1416                         while (j < PQntuples(tag)) {
1417                                 tag_id=atol(PQgetvalue(tag, j, 0));
1418                                 if (tag_id == id) {
1419                                         add_tag(PQgetvalue(tag, j, 1), PQgetvalue(tag, j, 2));
1420                                         tagged=1;
1421                                         j++;
1422                                 }
1423                                 if (tag_id < id)
1424                                         j++;
1425                                 if (tag_id > id)
1426                                         break;
1427                         }
1428                         if (tagged)
1429                                 end_node(out_nodes);
1430                         in_node=0;
1431                 }
1432                 PQclear(tag);
1433                 PQclear(node);
1434         }
1435         for (;;) {
1436                 way=PQexec(conn, "fetch 100000 from way");
1437                 if (! way) {
1438                         fprintf(stderr, "Cannot setup cursor for ways: %s\n", PQerrorMessage(conn));
1439                         PQclear(node);
1440                         exit(1);
1441                 }
1442                 count=PQntuples(way);
1443                 if (! count)
1444                         break;
1445                 min=atol(PQgetvalue(way, 0, 0));
1446                 max=atol(PQgetvalue(way, count-1, 0));
1447                 sprintf(query,"select way_id,node_id from way_node where way_id >= %ld and way_id <= %ld order by way_id,sequence_id", min, max);
1448                 node=PQexec(conn, query);
1449                 if (! node) {
1450                         fprintf(stderr, "Cannot query way_node: %s\n", PQerrorMessage(conn));
1451                         exit(1);
1452                 }
1453                 sprintf(query,"select way_id,name,value from way_tag where way_id >= %ld and way_id <= %ld order by way_id", min, max);
1454                 tag=PQexec(conn, query);
1455                 if (! tag) {
1456                         fprintf(stderr, "Cannot query way_tag: %s\n", PQerrorMessage(conn));
1457                         exit(1);
1458                 }
1459                 j=0;
1460                 k=0;
1461                 for (i = 0 ; i < count ; i++) {
1462                         id=atol(PQgetvalue(way, i, 0));
1463                         add_way(id);
1464                         tagged=0;
1465                         in_way=1;
1466                         processed_ways++;
1467                         while (k < PQntuples(node)) {
1468                                 node_id=atol(PQgetvalue(node, k, 0));
1469                                 if (node_id == id) {
1470                                         add_nd("",atol(PQgetvalue(node, k, 1)));
1471                                         tagged=1;
1472                                         k++;
1473                                 }
1474                                 if (node_id < id)
1475                                         k++;
1476                                 if (node_id > id)
1477                                         break;
1478                         }
1479                         while (j < PQntuples(tag)) {
1480                                 tag_id=atol(PQgetvalue(tag, j, 0));
1481                                 if (tag_id == id) {
1482                                         add_tag(PQgetvalue(tag, j, 1), PQgetvalue(tag, j, 2));
1483                                         tagged=1;
1484                                         j++;
1485                                 }
1486                                 if (tag_id < id)
1487                                         j++;
1488                                 if (tag_id > id)
1489                                         break;
1490                         }
1491                         if (tagged)
1492                                 end_way(out_ways);
1493                         in_way=0;
1494                 }
1495                 PQclear(tag);
1496                 PQclear(node);
1497                 PQclear(way);
1498         }
1499
1500         res=PQexec(conn, "commit");
1501         if (! res) {
1502                 fprintf(stderr, "Cannot commit transaction: %s\n", PQerrorMessage(conn));
1503                 PQclear(res);
1504                 exit(1);
1505         }
1506         sig_alrm(0);
1507         alarm(0);
1508         return 1;
1509 }
1510 #endif
1511
1512 static char buffer[200000];
1513
1514 static void
1515 phase1_map(struct map *map, FILE *out_ways, FILE *out_nodes)
1516 {
1517         struct map_rect *mr=map_rect_new(map, NULL);
1518         struct item *item;
1519         int count,max=16384;
1520         struct coord ca[max];
1521         struct item_bin ib;
1522         struct attr attr;
1523         FILE *file;
1524
1525         while ((item = map_rect_get_item(mr))) {
1526                 count=item_coord_get(item, ca, item->type < type_line ? 1: max);
1527                 item_buffer_set_type(buffer, item->type);
1528                 item_buffer_add_coord(buffer, ca, count);
1529                 while (item_attr_get(item, attr_any, &attr)) {
1530                         item_buffer_add_attr(buffer, &attr);
1531                 }
1532                 if (item->type >= type_line)
1533                         item_buffer_write(buffer, out_ways);
1534                 else
1535                         item_buffer_write(buffer, out_nodes);
1536         }
1537         map_rect_destroy(mr);
1538 }
1539
1540
1541 int bytes_read=0;
1542
1543 static struct item_bin *
1544 read_item(FILE *in)
1545 {
1546         struct item_bin *ib=(struct item_bin *) buffer;
1547         int r,s;
1548         r=fread(ib, sizeof(*ib), 1, in);
1549         if (r != 1)
1550                 return NULL;
1551         bytes_read+=r;
1552         assert((ib->len+1)*4 < sizeof(buffer));
1553         s=(ib->len+1)*4-sizeof(*ib);
1554         r=fread(ib+1, s, 1, in);
1555         if (r != 1)
1556                 return NULL;
1557         bytes_read+=r;
1558         return ib;
1559 }
1560
1561 static void
1562 bbox_extend(struct coord *c, struct rect *r)
1563 {
1564         if (c->x < r->l.x)
1565                 r->l.x=c->x;
1566         if (c->y < r->l.y)
1567                 r->l.y=c->y;
1568         if (c->x > r->h.x)
1569                 r->h.x=c->x;
1570         if (c->y > r->h.y)
1571                 r->h.y=c->y;
1572 }
1573
1574 static void
1575 bbox(struct coord *c, int count, struct rect *r)
1576 {
1577         if (! count)
1578                 return;
1579         r->l=*c;
1580         r->h=*c;
1581         while (--count) {
1582                 c++;
1583                 bbox_extend(c, r);
1584         }
1585 }
1586
1587 static int
1588 contains_bbox(int xl, int yl, int xh, int yh, struct rect *r)
1589 {
1590         if (r->h.x < xl || r->h.x > xh) {
1591                 return 0;
1592         }
1593         if (r->l.x > xh || r->l.x < xl) {
1594                 return 0;
1595         }
1596         if (r->h.y < yl || r->h.y > yh) {
1597                 return 0;
1598         }
1599         if (r->l.y > yh || r->l.y < yl) {
1600                 return 0;
1601         }
1602         return 1;
1603
1604 }
1605 struct rect world_bbox = {
1606         { -20000000, -20000000},
1607         {  20000000,  20000000},
1608 };
1609
1610 static void
1611 tile(struct rect *r, char *ret, int max)
1612 {
1613         int x0,x1,x2,x3,x4;
1614         int y0,y1,y2,y3,y4;
1615         int i;
1616         x0=world_bbox.l.x;
1617         y0=world_bbox.l.y;
1618         x4=world_bbox.h.x;
1619         y4=world_bbox.h.y;
1620         for (i = 0 ; i < max ; i++) {
1621                 x2=(x0+x4)/2;
1622                 y2=(y0+y4)/2;
1623                 x1=(x0+x2)/2;
1624                 y1=(y0+y2)/2;
1625                 x3=(x2+x4)/2;
1626                 y3=(y2+y4)/2;
1627                 if (     contains_bbox(x0,y0,x2,y2,r)) {
1628                         strcat(ret,"d");
1629                         x4=x2;
1630                         y4=y2;
1631                 } else if (contains_bbox(x2,y0,x4,y2,r)) {
1632                         strcat(ret,"c");
1633                         x0=x2;
1634                         y4=y2;
1635                 } else if (contains_bbox(x0,y2,x2,y4,r)) {
1636                         strcat(ret,"b");
1637                         x4=x2;
1638                         y0=y2;
1639                 } else if (contains_bbox(x2,y2,x4,y4,r)) {
1640                         strcat(ret,"a");
1641                         x0=x2;
1642                         y0=y2;
1643                 } else
1644                         return;
1645         }
1646 }
1647
1648 static void
1649 tile_bbox(char *tile, struct rect *r)
1650 {
1651         struct coord c;
1652         *r=world_bbox;
1653         while (*tile) {
1654                 c.x=(r->l.x+r->h.x)/2;
1655                 c.y=(r->l.y+r->h.y)/2;
1656                 switch (*tile) {
1657                 case 'a':
1658                         r->l.x=c.x;
1659                         r->l.y=c.y;
1660                         break;
1661                 case 'b':
1662                         r->h.x=c.x;
1663                         r->l.y=c.y;
1664                         break;
1665                 case 'c':
1666                         r->l.x=c.x;
1667                         r->h.y=c.y;
1668                         break;
1669                 case 'd':
1670                         r->h.x=c.x;
1671                         r->h.y=c.y;
1672                         break;
1673                 }
1674                 tile++;
1675         }
1676 }
1677
1678 GHashTable *tile_hash;
1679 GHashTable *tile_hash2;
1680
1681 static void
1682 tile_extend(char *tile, struct item_bin *ib, GList **tiles_list)
1683 {
1684         struct tile_head *th=NULL;
1685         if (debug_tile(tile))
1686                 fprintf(stderr,"Tile:Writing %d bytes to '%s' (%p,%p)\n", (ib->len+1)*4, tile, g_hash_table_lookup(tile_hash, tile), tile_hash2 ? g_hash_table_lookup(tile_hash2, tile) : NULL);
1687         if (tile_hash2)
1688                 th=g_hash_table_lookup(tile_hash2, tile);
1689         if (!th)
1690                 th=g_hash_table_lookup(tile_hash, tile);
1691         if (! th) {
1692                 th=malloc(sizeof(struct tile_head)+ sizeof( char* ) );
1693                 assert(th != NULL);
1694                 // strcpy(th->subtiles, tile);
1695                 th->num_subtiles=1;
1696                 th->total_size=0;
1697                 th->total_size_used=0;
1698                 th->zipnum=0;
1699                 th->zip_data=NULL;
1700                 th->name=string_hash_lookup(tile);
1701                 *th_get_subtile( th, 0 ) = th->name;
1702
1703                 if (tile_hash2)
1704                         g_hash_table_insert(tile_hash2, string_hash_lookup( th->name ), th);
1705                 if (tiles_list)
1706                         *tiles_list=g_list_append(*tiles_list, string_hash_lookup( th->name ) );
1707                 processed_tiles++;
1708                 if (debug_tile(tile))
1709                         fprintf(stderr,"new '%s'\n", tile);
1710         }
1711         th->total_size+=ib->len*4+4;
1712         if (debug_tile(tile))
1713                 fprintf(stderr,"New total size of %s(%p):%d\n", th->name, th, th->total_size);
1714         g_hash_table_insert(tile_hash, string_hash_lookup( th->name ), th);
1715 }
1716
1717 static int
1718 tile_data_size(char *tile)
1719 {
1720         struct tile_head *th;
1721         th=g_hash_table_lookup(tile_hash, tile);
1722         if (! th)
1723                 return 0;
1724         return th->total_size;
1725 }
1726
1727 static int
1728 merge_tile(char *base, char *sub)
1729 {
1730         struct tile_head *thb, *ths;
1731         thb=g_hash_table_lookup(tile_hash, base);
1732         ths=g_hash_table_lookup(tile_hash, sub);
1733         if (! ths)
1734                 return 0;
1735         if (debug_tile(base) || debug_tile(sub))
1736                 fprintf(stderr,"merging '%s'(%p) (%d) with '%s'(%p) (%d)\n", base, thb, thb ? thb->total_size : 0, sub, ths, ths->total_size);
1737         if (! thb) {
1738                 thb=ths;
1739                 g_hash_table_remove(tile_hash, sub);
1740                 thb->name=string_hash_lookup(base);
1741                 g_hash_table_insert(tile_hash, string_hash_lookup( thb->name ), thb);
1742
1743         } else {
1744                 thb=realloc(thb, sizeof(struct tile_head)+( ths->num_subtiles+thb->num_subtiles ) * sizeof( char*) );
1745                 assert(thb != NULL);
1746                 memcpy( th_get_subtile( thb, thb->num_subtiles ), th_get_subtile( ths, 0 ), ths->num_subtiles * sizeof( char*) );
1747                 thb->num_subtiles+=ths->num_subtiles;
1748                 thb->total_size+=ths->total_size;
1749                 g_hash_table_insert(tile_hash, string_hash_lookup( thb->name ), thb);
1750                 g_hash_table_remove(tile_hash, sub);
1751                 g_free(ths);
1752         }
1753         return 1;
1754 }
1755
1756
1757 static void
1758 get_tiles_list_func(char *key, struct tile_head *th, GList **list)
1759 {
1760         *list=g_list_prepend(*list, key);
1761 }
1762
1763 static GList *
1764 get_tiles_list(void)
1765 {
1766         GList *ret=NULL;
1767         g_hash_table_foreach(tile_hash, (GHFunc)get_tiles_list_func, &ret);
1768         return ret;
1769 }
1770
1771 #if 0
1772 static void
1773 write_tile(char *key, struct tile_head *th, gpointer dummy)
1774 {
1775         FILE *f;
1776         char buffer[1024];
1777         fprintf(stderr,"DEBUG: Writing %s\n", key);
1778         strcpy(buffer,"tiles/");
1779         strcat(buffer,key);
1780 #if 0
1781         strcat(buffer,".bin");
1782 #endif
1783         f=fopen(buffer, "wb+");
1784         while (th) {
1785                 fwrite(th->data, th->size, 1, f);
1786                 th=th->next;
1787         }
1788         fclose(f);
1789 }
1790 #endif
1791
1792 static void
1793 write_item(char *tile, struct item_bin *ib)
1794 {
1795         struct tile_head *th;
1796         int size;
1797
1798         th=g_hash_table_lookup(tile_hash2, tile);
1799         if (! th)
1800                 th=g_hash_table_lookup(tile_hash, tile);
1801         if (th) {
1802                 if (th->process != 0 && th->process != 1) {
1803                         fprintf(stderr,"error with tile '%s' of length %d\n", tile, strlen(tile));
1804                         abort();
1805                 }
1806                 if (! th->process)
1807                         return;
1808                 if (debug_tile(tile))
1809                         fprintf(stderr,"Data:Writing %d bytes to '%s' (%p,%p)\n", (ib->len+1)*4, tile, g_hash_table_lookup(tile_hash, tile), tile_hash2 ? g_hash_table_lookup(tile_hash2, tile) : NULL);
1810                 size=(ib->len+1)*4;
1811                 if (th->total_size_used+size > th->total_size) {
1812                         fprintf(stderr,"Overflow in tile %s (used %d max %d item %d)\n", tile, th->total_size_used, th->total_size, size);
1813                         exit(1);
1814                         return;
1815                 }
1816                 memcpy(th->zip_data+th->total_size_used, ib, size);
1817                 th->total_size_used+=size;
1818         } else {
1819                 fprintf(stderr,"no tile hash found for %s\n", tile);
1820                 exit(1);
1821         }
1822 }
1823
1824 static void
1825 write_item_part(FILE *out, struct item_bin *orig, int first, int last)
1826 {
1827         struct item_bin new;
1828         struct coord *c=(struct coord *)(orig+1);
1829         char *attr=(char *)(c+orig->clen/2);
1830         int attr_len=orig->len-orig->clen-2;
1831         processed_ways++;
1832         new.type=orig->type;
1833         new.clen=(last-first+1)*2;
1834         new.len=new.clen+attr_len+2;
1835 #if 0
1836         fprintf(stderr,"first %d last %d type 0x%x len %d clen %d attr_len %d\n", first, last, new.type, new.len, new.clen, attr_len);
1837 #endif
1838         fwrite(&new, sizeof(new), 1, out);
1839         fwrite(c+first, new.clen*4, 1, out);
1840         fwrite(attr, attr_len*4, 1, out);
1841 }
1842
1843 static int
1844 phase2(FILE *in, FILE *out)
1845 {
1846         struct coord *c;
1847         int i,ccount,last,ndref;
1848         struct item_bin *ib;
1849         struct node_item *ni;
1850
1851         processed_nodes=processed_nodes_out=processed_ways=processed_relations=processed_tiles=0;
1852         sig_alrm(0);
1853         while ((ib=read_item(in))) {
1854 #if 0
1855                 fprintf(stderr,"type 0x%x len %d clen %d\n", ib->type, ib->len, ib->clen);
1856 #endif
1857                 ccount=ib->clen/2;
1858                 c=(struct coord *)(ib+1);
1859                 last=0;
1860                 for (i = 0 ; i < ccount ; i++) {
1861                         if (IS_REF(c[i])) {
1862                                 ndref=REF(c[i]);
1863                                 ni=node_item_get(ndref);
1864 #if 0
1865                                 fprintf(stderr,"ni=%p\n", ni);
1866 #endif
1867                                 c[i]=ni->c;
1868                                 if (ni->ref_way > 1 && i != 0 && i != ccount-1 && item_is_street(*ib)) {
1869                                         write_item_part(out, ib, last, i);
1870                                         last=i;
1871                                 }
1872                         }
1873                 }
1874                 write_item_part(out, ib, last, ccount-1);
1875         }
1876         sig_alrm(0);
1877         alarm(0);
1878         return 0;
1879 }
1880
1881 static void
1882 phase34_process_file(int phase, FILE *in)
1883 {
1884         struct item_bin *ib;
1885         struct rect r;
1886         char buffer[1024];
1887         int max;
1888
1889         while ((ib=read_item(in))) {
1890                 if (ib->type < 0x80000000)
1891                         processed_nodes++;
1892                 else
1893                         processed_ways++;
1894                 bbox((struct coord *)(ib+1), ib->clen/2, &r);
1895                 buffer[0]='\0';
1896                 max=14;
1897                 if (ib->type == type_street_n_lanes || ib->type == type_highway_city || ib->type == type_highway_land || ib->type == type_ramp)
1898                         max=8;
1899                 if (ib->type == type_street_3_city || ib->type == type_street_4_city || ib->type == type_street_3_land || ib->type == type_street_4_land)
1900                         max=12;
1901
1902                 tile(&r, buffer, max);
1903 #if 0
1904                 fprintf(stderr,"%s\n", buffer);
1905 #endif
1906                 if (phase == 3)
1907                         tile_extend(buffer, ib, NULL);
1908                 else
1909                         write_item(buffer, ib);
1910         }
1911 }
1912
1913 struct index_item {
1914         struct item_bin item;
1915         struct rect r;
1916         struct attr_bin attr_order;
1917         short min;
1918         short max;
1919         struct attr_bin attr_zipfile_ref;
1920         int zipfile_ref;
1921 };
1922
1923 static void
1924 index_submap_add(int phase, struct tile_head *th, GList **tiles_list)
1925 {
1926         struct index_item ii;
1927         int len=strlen(th->name);
1928         char index_tile[len+1];
1929
1930         ii.min=(len > 4) ? len-4 : 0;
1931         ii.max=255;
1932         strcpy(index_tile, th->name);
1933         if (len > 6)
1934                 len=6;
1935         else
1936                 len=0;
1937         index_tile[len]=0;
1938         tile_bbox(th->name, &ii.r);
1939
1940         ii.item.len=sizeof(ii)/4-1;
1941         ii.item.type=type_submap;
1942         ii.item.clen=4;
1943
1944         ii.attr_order.len=2;
1945         ii.attr_order.type=attr_order;
1946
1947         ii.attr_zipfile_ref.len=2;
1948         ii.attr_zipfile_ref.type=attr_zipfile_ref;
1949         ii.zipfile_ref=th->zipnum;
1950
1951         if (phase == 3)
1952                 tile_extend(index_tile, (struct item_bin *)&ii, tiles_list);
1953         else
1954                 write_item(index_tile, (struct item_bin *)&ii);
1955 #if 0
1956         unsigned int *c=(unsigned int *)&ii;
1957         int i;
1958         for (i = 0 ; i < sizeof(ii)/4 ; i++) {
1959                 fprintf(stderr,"%08x ", c[i]);
1960         }
1961         fprintf(stderr,"\n");
1962 #endif
1963 }
1964
1965 static int
1966 add_tile_hash(struct tile_head *th)
1967 {
1968         int idx,len,maxnamelen=0;
1969         char **data;
1970
1971 #if 0
1972         g_hash_table_insert(tile_hash2, string_hash_lookup( th->name ), th);
1973 #endif
1974         for( idx = 0; idx < th->num_subtiles; idx++ ) {
1975
1976         data = th_get_subtile( th, idx );
1977
1978                 if (debug_tile(data) || debug_tile(th->name)) {
1979                         fprintf(stderr,"Parent for '%s' is '%s'\n", *data, th->name);
1980                 }
1981
1982                 g_hash_table_insert(tile_hash2, *data, th);
1983
1984                 len = strlen( *data );
1985
1986                 if (len > maxnamelen) {
1987                         maxnamelen=len;
1988                 }
1989         }
1990         return maxnamelen;
1991 }
1992
1993
1994 static int
1995 create_tile_hash(void)
1996 {
1997         struct tile_head *th;
1998         int len,maxnamelen=0;
1999
2000         tile_hash2=g_hash_table_new(g_str_hash, g_str_equal);
2001         th=tile_head_root;
2002         while (th) {
2003                 len=add_tile_hash(th);
2004                 if (len > maxnamelen)
2005                         maxnamelen=len;
2006                 th=th->next;
2007         }
2008         return maxnamelen;
2009 }
2010
2011 static void
2012 create_tile_hash_list(GList *list)
2013 {
2014         GList *next;
2015         struct tile_head *th;
2016
2017         tile_hash2=g_hash_table_new(g_str_hash, g_str_equal);
2018
2019         fprintf(stderr,"list=%p\n", list);
2020         next=g_list_first(list);
2021         while (next) {
2022                 th=g_hash_table_lookup(tile_hash, next->data);
2023                 if (!th) {
2024                         fprintf(stderr,"No tile found for '%s'\n", (char *)(next->data));
2025                 }
2026                 add_tile_hash(th);
2027                 next=g_list_next(next);
2028         }
2029 }
2030
2031 static void
2032 destroy_tile_hash(void)
2033 {
2034         g_hash_table_destroy(tile_hash2);
2035         tile_hash2=NULL;
2036 }
2037
2038 static int zipnum;
2039
2040 static void write_countrydir(int phase, int maxnamelen);
2041
2042 static void
2043 write_tilesdir(int phase, int maxlen, FILE *out)
2044 {
2045         int idx,len;
2046         GList *tiles_list,*next;
2047         char **data;
2048         struct tile_head *th,**last=NULL;
2049         zipnum=0;
2050
2051         tiles_list=get_tiles_list();
2052         if (phase == 3)
2053                 create_tile_hash_list(tiles_list);
2054         next=g_list_first(tiles_list);
2055         last=&tile_head_root;
2056         if (! maxlen) {
2057                 while (next) {
2058                         if (strlen(next->data) > maxlen)
2059                                 maxlen=strlen(next->data);
2060                         next=g_list_next(next);
2061                 }
2062         }
2063         len=maxlen;
2064         while (len >= 0) {
2065 #if 0
2066                 fprintf(stderr,"PROGRESS: collecting tiles with len=%d\n", len);
2067 #endif
2068 #ifdef GENERATE_INDEX
2069                 if (! len)
2070                         write_countrydir(phase, maxlen);
2071 #endif
2072                 next=g_list_first(tiles_list);
2073                 while (next) {
2074                         if (strlen(next->data) == len) {
2075                                 th=g_hash_table_lookup(tile_hash, next->data);
2076                                 if (phase == 3) {
2077                                         *last=th;
2078                                         last=&th->next;
2079                                         th->next=NULL;
2080                                         th->zipnum=zipnum;
2081                                         fprintf(out,"%s:%d",(char *)next->data,th->total_size);
2082
2083                                         for ( idx = 0; idx< th->num_subtiles; idx++ ){
2084                         data= th_get_subtile( th, idx );
2085                                                 fprintf(out,":%s", *data);
2086                                         }
2087
2088                                         fprintf(out,"\n");
2089                                 }
2090                                 if (th->name[0])
2091                                         index_submap_add(phase, th, &tiles_list);
2092                                 zipnum++;
2093                                 processed_tiles++;
2094                         }
2095                         next=g_list_next(next);
2096                 }
2097                 len--;
2098         }
2099 }
2100
2101 static void
2102 merge_tiles(void)
2103 {
2104         struct tile_head *th;
2105         char basetile[1024];
2106         char subtile[1024];
2107         GList *tiles_list_sorted,*last;
2108         int i,i_min,len,size_all,size[5],size_min,work_done;
2109         long long zip_size;
2110
2111         do {
2112                 tiles_list_sorted=get_tiles_list();
2113                 fprintf(stderr,"PROGRESS: sorting %d tiles\n", g_list_length(tiles_list_sorted));
2114                 tiles_list_sorted=g_list_sort(tiles_list_sorted, (GCompareFunc)strcmp);
2115                 fprintf(stderr,"PROGRESS: sorting %d tiles done\n", g_list_length(tiles_list_sorted));
2116                 last=g_list_last(tiles_list_sorted);
2117                 zip_size=0;
2118                 while (last) {
2119                         th=g_hash_table_lookup(tile_hash, last->data);
2120                         zip_size+=th->total_size;
2121                         last=g_list_previous(last);
2122                 }
2123                 fprintf(stderr,"DEBUG: size=%Ld\n", zip_size);
2124                 last=g_list_last(tiles_list_sorted);
2125                 work_done=0;
2126                 while (last) {
2127                         processed_tiles++;
2128                         len=strlen(last->data);
2129                         if (len >= 1) {
2130                                 strcpy(basetile,last->data);
2131                                 basetile[len-1]='\0';
2132                                 strcpy(subtile,last->data);
2133                                 for (i = 0 ; i < 4 ; i++) {
2134                                         subtile[len-1]='a'+i;
2135                                         size[i]=tile_data_size(subtile);
2136                                 }
2137                                 size[4]=tile_data_size(basetile);
2138                                 size_all=size[0]+size[1]+size[2]+size[3]+size[4];
2139                                 if (size_all < 65536 && size_all > 0 && size_all != size[4]) {
2140                                         for (i = 0 ; i < 4 ; i++) {
2141                                                 subtile[len-1]='a'+i;
2142                                                 work_done+=merge_tile(basetile, subtile);
2143                                         }
2144                                 } else {
2145                                         for (;;) {
2146                                                 size_min=size_all;
2147                                                 i_min=-1;
2148                                                 for (i = 0 ; i < 4 ; i++) {
2149                                                         if (size[i] && size[i] < size_min) {
2150                                                                 size_min=size[i];
2151                                                                 i_min=i;
2152                                                         }
2153                                                 }
2154                                                 if (i_min == -1)
2155                                                         break;
2156                                                 if (size[4]+size_min >= 65536)
2157                                                         break;
2158                                                 subtile[len-1]='a'+i_min;
2159                                                 work_done+=merge_tile(basetile, subtile);
2160                                                 size[4]+=size[i_min];
2161                                                 size[i_min]=0;
2162                                         }
2163                                 }
2164                         }
2165                         last=g_list_previous(last);
2166                 }
2167                 g_list_free(tiles_list_sorted);
2168                 fprintf(stderr,"PROGRESS: merged %d tiles\n", work_done);
2169         } while (work_done);
2170 }
2171
2172 struct country_index_item {
2173         struct item_bin item;
2174         struct attr_bin attr_country_id;
2175         int country_id;
2176         struct attr_bin attr_zipfile_ref;
2177         int zipfile_ref;
2178 };
2179
2180 static void
2181 index_country_add(int phase, int country_id, int zipnum)
2182 {
2183         struct country_index_item ii;
2184         char *index_tile="";
2185
2186         ii.item.len=sizeof(ii)/4-1;
2187         ii.item.type=type_countryindex;
2188         ii.item.clen=0;
2189
2190         ii.attr_country_id.len=2;
2191         ii.attr_country_id.type=attr_country_id;
2192         ii.country_id=country_id;
2193
2194         ii.attr_zipfile_ref.len=2;
2195         ii.attr_zipfile_ref.type=attr_zipfile_ref;
2196         ii.zipfile_ref=zipnum;
2197
2198         if (phase == 3)
2199                 tile_extend(index_tile, (struct item_bin *)&ii, NULL);
2200         else
2201                 write_item(index_tile, (struct item_bin *)&ii);
2202 }
2203
2204 #ifdef GENERATE_INDEX
2205 struct aux_tile {
2206         char *name;
2207         char *filename;
2208         int size;
2209 };
2210
2211 static int
2212 add_aux_tile(int phase, char *name, char *filename, int size)
2213 {
2214         struct aux_tile *at;
2215         if (phase == 3) {
2216                 at=g_new0(struct aux_tile, 1);
2217                 at->name=g_strdup(name);
2218                 at->filename=g_strdup(filename);
2219                 at->size=size;
2220                 aux_tile_list=g_list_append(aux_tile_list, at);
2221         }
2222         return zipnum++;
2223 }
2224
2225 static int
2226 write_aux_tiles(FILE *out, FILE *dir_out, int compression_level, int namelen)
2227 {
2228         GList *l=aux_tile_list;
2229         struct aux_tile *at;
2230         char *buffer;
2231         FILE *f;
2232         int count=0;
2233
2234         while (l) {
2235                 at=l->data;
2236                 buffer=malloc(at->size);
2237                 assert(buffer != NULL);
2238                 f=fopen(at->filename,"r");
2239                 assert(f != NULL);
2240                 fread(buffer, at->size, 1, f);
2241                 fclose(f);
2242                 write_zipmember(out, dir_out, at->name, namelen, buffer, at->size, compression_level);
2243                 count++;
2244                 l=g_list_next(l);
2245         }
2246         return count;
2247 }
2248
2249 static void
2250 write_countrydir(int phase, int maxnamelen)
2251 {
2252         int i,zipnum;
2253         int max=11;
2254         char tilename[32];
2255         char searchtile[32];
2256         char filename[32];
2257         struct country_table *co;
2258         for (i = 0 ; i < sizeof(country_table)/sizeof(struct country_table) ; i++) {
2259                 co=&country_table[i];
2260                 if (co->size) {
2261                         tilename[0]='\0';
2262                         tile(&co->r, tilename, max);
2263                         sprintf(searchtile,"%ss%d", tilename, 0);
2264                         sprintf(filename,"country_%d.bin", co->countryid);
2265                         zipnum=add_aux_tile(phase, searchtile, filename, co->size);
2266                         index_country_add(phase,co->countryid,zipnum);
2267                 }
2268         }
2269 }
2270
2271 static void
2272 remove_countryfiles(void)
2273 {
2274         int i;
2275         char filename[32];
2276         struct country_table *co;
2277
2278         for (i = 0 ; i < sizeof(country_table)/sizeof(struct country_table) ; i++) {
2279                 co=&country_table[i];
2280                 if (co->size) {
2281                         sprintf(filename,"country_%d.bin", co->countryid);
2282                         unlink(filename);
2283                 }
2284         }
2285 }
2286 #endif
2287
2288 static int
2289 phase34(int phase, int maxnamelen, FILE *ways_in, FILE *nodes_in, FILE *tilesdir_out)
2290 {
2291
2292         processed_nodes=processed_nodes_out=processed_ways=processed_relations=processed_tiles=0;
2293         bytes_read=0;
2294         sig_alrm(0);
2295         if (phase == 3)
2296                 tile_hash=g_hash_table_new(g_str_hash, g_str_equal);
2297         if (ways_in)
2298                 phase34_process_file(phase, ways_in);
2299         if (nodes_in)
2300                 phase34_process_file(phase, nodes_in);
2301         fprintf(stderr,"read %d bytes\n", bytes_read);
2302         if (phase == 3)
2303                 merge_tiles();
2304         sig_alrm(0);
2305         alarm(0);
2306         write_tilesdir(phase, maxnamelen, tilesdir_out);
2307
2308         return 0;
2309
2310 }
2311
2312 static void
2313 dump_coord(struct coord *c)
2314 {
2315         printf("0x%x 0x%x",c->x, c->y);
2316 }
2317
2318 static void
2319 dump(FILE *in)
2320 {
2321         struct item_bin *ib;
2322         struct coord *c;
2323         struct attr_bin *a;
2324         struct attr attr;
2325         int *attr_start;
2326         int *attr_end;
2327         int i;
2328         char *str;
2329         while ((ib=read_item(in))) {
2330                 c=(struct coord *)(ib+1);
2331                 if (ib->type < type_line) {
2332                         dump_coord(c);
2333                         printf(" ");
2334                 }
2335                 attr_start=(int *)(ib+1)+ib->clen;
2336                 attr_end=(int *)ib+ib->len+1;
2337                 printf("type=%s", item_to_name(ib->type));
2338                 while (attr_start < attr_end) {
2339                         a=(struct attr_bin *)(attr_start);
2340                         attr_start+=a->len+1;
2341                         attr.type=a->type;
2342                         attr_data_set(&attr, (a+1));
2343                         str=attr_to_text(&attr, NULL, 1);
2344                         printf(" %s=\"%s\"", attr_to_name(a->type), str);
2345                         g_free(str);
2346                 }
2347                 printf(" debug=\"length=%d\"", ib->len);
2348                 printf("\n");
2349                 if (ib->type >= type_line) {
2350                         for (i = 0 ; i < ib->clen/2 ; i++) {
2351                                 dump_coord(c+i);
2352                                 printf("\n");
2353                         }
2354
2355                 }
2356         }
2357 }
2358
2359 static int
2360 phase3(FILE *ways_in, FILE *nodes_in, FILE *tilesdir_out)
2361 {
2362         return phase34(3, 0, ways_in, nodes_in, tilesdir_out);
2363 }
2364
2365 static long long zipoffset;
2366 static int zipdir_size;
2367
2368 static int
2369 compress2_int(Byte *dest, uLongf *destLen, const Bytef *source, uLong sourceLen, int level)
2370 {
2371         z_stream stream;
2372         int err;
2373
2374         stream.next_in = (Bytef*)source;
2375         stream.avail_in = (uInt)sourceLen;
2376         stream.next_out = dest;
2377         stream.avail_out = (uInt)*destLen;
2378         if ((uLong)stream.avail_out != *destLen) return Z_BUF_ERROR;
2379
2380         stream.zalloc = (alloc_func)0;
2381         stream.zfree = (free_func)0;
2382         stream.opaque = (voidpf)0;
2383
2384         err = deflateInit2(&stream, level, Z_DEFLATED, -15, 9, Z_DEFAULT_STRATEGY);
2385         if (err != Z_OK) return err;
2386
2387         err = deflate(&stream, Z_FINISH);
2388         if (err != Z_STREAM_END) {
2389                 deflateEnd(&stream);
2390                 return err == Z_OK ? Z_BUF_ERROR : err;
2391         }
2392         *destLen = stream.total_out;
2393
2394         err = deflateEnd(&stream);
2395         return err;
2396 }
2397
2398 static void
2399 write_zipmember(FILE *out, FILE *dir_out, char *name, int filelen, char *data, int data_size, int compression_level)
2400 {
2401         struct zip_lfh lfh = {
2402                 0x04034b50,
2403                 0x0a,
2404                 0x0,
2405                 0x0,
2406                 0xbe2a,
2407                 0x5d37,
2408                 0x0,
2409                 0x0,
2410                 0x0,
2411                 filelen,
2412                 0x0,
2413         };
2414         struct zip_cd cd = {
2415                 0x02014b50,
2416                 0x17,
2417                 0x00,
2418                 0x0a,
2419                 0x00,
2420                 0x0000,
2421                 0x0,
2422                 0xbe2a,
2423                 0x5d37,
2424                 0x0,
2425                 0x0,
2426                 0x0,
2427                 filelen,
2428                 0x0000,
2429                 0x0000,
2430                 0x0000,
2431                 0x0000,
2432                 0x0,
2433                 zipoffset,
2434         };
2435         char filename[filelen+1];
2436         int error,crc,len,comp_size=data_size;
2437         uLongf destlen=data_size+data_size/500+12;
2438         char compbuffer[destlen];
2439
2440         crc=crc32(0, NULL, 0);
2441         crc=crc32(crc, (unsigned char *)data, data_size);
2442 #ifdef HAVE_ZLIB
2443         if (compression_level) {
2444                 error=compress2_int((Byte *)compbuffer, &destlen, (Bytef *)data, data_size, compression_level);
2445                 if (error == Z_OK) {
2446                         if (destlen < data_size) {
2447                                 data=compbuffer;
2448                                 comp_size=destlen;
2449                         }
2450                 } else {
2451                         fprintf(stderr,"compress2 returned %d\n", error);
2452                 }
2453         }
2454 #endif
2455         lfh.zipcrc=crc;
2456         lfh.zipsize=comp_size;
2457         lfh.zipuncmp=data_size;
2458         lfh.zipmthd=compression_level ? 8:0;
2459         cd.zipccrc=crc;
2460         cd.zipcsiz=comp_size;
2461         cd.zipcunc=data_size;
2462         cd.zipcmthd=compression_level ? 8:0;
2463         strcpy(filename, name);
2464         len=strlen(filename);
2465         while (len < filelen) {
2466                 filename[len++]='_';
2467         }
2468         filename[filelen]='\0';
2469         fwrite(&lfh, sizeof(lfh), 1, out);
2470         fwrite(filename, filelen, 1, out);
2471         fwrite(data, comp_size, 1, out);
2472         zipoffset+=sizeof(lfh)+filelen+comp_size;
2473         fwrite(&cd, sizeof(cd), 1, dir_out);
2474         fwrite(filename, filelen, 1, dir_out);
2475         zipdir_size+=sizeof(cd)+filelen;
2476 }
2477
2478 static int
2479 process_slice(FILE *ways_in, FILE *nodes_in, int size, int maxnamelen, FILE *out, FILE *dir_out, int compression_level)
2480 {
2481         struct tile_head *th;
2482         char *slice_data,*zip_data;
2483         int zipfiles=0;
2484
2485         slice_data=malloc(size);
2486         assert(slice_data != NULL);
2487         zip_data=slice_data;
2488         th=tile_head_root;
2489         while (th) {
2490                 if (th->process) {
2491                         th->zip_data=zip_data;
2492                         zip_data+=th->total_size;
2493                 }
2494                 th=th->next;
2495         }
2496         if (ways_in)
2497                 fseek(ways_in, 0, SEEK_SET);
2498         if (nodes_in)
2499                 fseek(nodes_in, 0, SEEK_SET);
2500         phase34(4, maxnamelen, ways_in, nodes_in, NULL);
2501
2502         th=tile_head_root;
2503         while (th) {
2504                 if (th->process) {
2505 #ifdef GENERATE_INDEX
2506                         if (! strlen(th->name))
2507                                 zipfiles+=write_aux_tiles(out, dir_out, compression_level, maxnamelen);
2508 #endif
2509                         if (th->total_size != th->total_size_used) {
2510                                 fprintf(stderr,"Size error '%s': %d vs %d\n", th->name, th->total_size, th->total_size_used);
2511                                 exit(1);
2512                         } else {
2513                                 if (strlen(th->name))
2514                                         write_zipmember(out, dir_out, th->name, maxnamelen, th->zip_data, th->total_size, compression_level);
2515                                 else {
2516                                         write_zipmember(out, dir_out, "index", sizeof("index")-1, th->zip_data, th->total_size, compression_level);
2517                                 }
2518                                 zipfiles++;
2519                         }
2520                 }
2521                 th=th->next;
2522         }
2523         free(slice_data);
2524
2525         return zipfiles;
2526 }
2527
2528 static void
2529 cat(FILE *in, FILE *out)
2530 {
2531         size_t size;
2532         char buffer[4096];
2533         while ((size=fread(buffer, 1, 4096, in)))
2534                 fwrite(buffer, 1, size, out);
2535 }
2536
2537 static int
2538 phase4(FILE *ways_in, FILE *nodes_in, FILE *out, FILE *dir_out, int compression_level)
2539 {
2540         int slice_size=1024*1024*1024;
2541         int maxnamelen,size,slices;
2542         int zipfiles=0;
2543         struct tile_head *th,*th2;
2544         struct zip_eoc eoc = {
2545                 0x06054b50,
2546                 0x0000,
2547                 0x0000,
2548                 0x0000,
2549                 0x0000,
2550                 0x0,
2551                 0x0,
2552                 0x0,
2553         };
2554
2555         maxnamelen=create_tile_hash();
2556
2557         th=tile_head_root;
2558         size=0;
2559         slices=0;
2560         fprintf(stderr, "Maximum slice size %d\n", slice_size);
2561         while (th) {
2562                 if (size + th->total_size > slice_size) {
2563                         fprintf(stderr,"Slice %d is of size %d\n", slices, size);
2564                         size=0;
2565                         slices++;
2566                 }
2567                 size+=th->total_size;
2568                 th=th->next;
2569         }
2570         if (size)
2571                 fprintf(stderr,"Slice %d is of size %d\n", slices, size);
2572         th=tile_head_root;
2573         size=0;
2574         slices=0;
2575         while (th) {
2576                 th2=tile_head_root;
2577                 while (th2) {
2578                         th2->process=0;
2579                         th2=th2->next;
2580                 }
2581                 size=0;
2582                 while (th && size+th->total_size < slice_size) {
2583                         size+=th->total_size;
2584                         th->process=1;
2585                         th=th->next;
2586                 }
2587                 zipfiles+=process_slice(ways_in, nodes_in, size, maxnamelen, out, dir_out, compression_level);
2588                 slices++;
2589         }
2590         fseek(dir_out, 0, SEEK_SET);
2591         cat(dir_out, out);
2592         eoc.zipenum=zipfiles;
2593         eoc.zipecenn=zipfiles;
2594         eoc.zipecsz=zipdir_size;
2595         eoc.zipeofst=zipoffset;
2596         fwrite(&eoc, sizeof(eoc), 1, out);
2597         sig_alrm(0);
2598         alarm(0);
2599         return 0;
2600 }
2601
2602 static void
2603 usage(FILE *f)
2604 {
2605         /* DEVELOPPERS : don't forget to update the manpage if you modify theses options */
2606         fprintf(f,"\n");
2607         fprintf(f,"osm2navit - parse osm textfile and converts to NavIt binfile format\n\n");
2608         fprintf(f,"Usage :\n");
2609         fprintf(f,"bzcat planet.osm.bz2 | osm2navit mymap.bin\n");
2610         fprintf(f,"Available switches:\n");
2611         fprintf(f,"-h (--help)              : this screen\n");
2612         fprintf(f,"-N (--nodes-only)        : process only nodes\n");
2613         fprintf(f,"-W (--ways-only)         : process only ways\n");
2614         fprintf(f,"-a (--attr-debug-level)  : control which data is included in the debug attribute\n");
2615         fprintf(f,"-c (--dump-coordinates)  : dump coordinates after phase 1\n");
2616 #ifdef HAVE_POSTGRESQL
2617         fprintf(f,"-d (--db)                : get osm data out of a postgresql database with osm simple scheme and given connect string\n");
2618 #endif
2619         fprintf(f,"-e (--end)               : end at specified phase\n");
2620         fprintf(f,"-k (--keep-tmpfiles)     : do not delete tmp files after processing. useful to reuse them\n\n");
2621         fprintf(f,"-o (--coverage)          : map every street to item overage\n");
2622         fprintf(f,"-s (--start)             : start at specified phase\n");
2623         fprintf(f,"-i (--input-file)        : specify the input file name (OSM), overrules default stdin\n");
2624         fprintf(f,"-w (--dedupe-ways)       : ensure no duplicate ways or nodes. useful when using several input files\n");
2625         fprintf(f,"-z (--compression-level) : set the compression level\n");
2626         exit(1);
2627 }
2628
2629 static void
2630 process_binfile(FILE *in, FILE *out)
2631 {
2632         struct item_bin *ib;
2633         while ((ib=read_item(in))) {
2634                 fwrite(ib, (ib->len+1)*4, 1, out);
2635         }
2636 }
2637
2638 int main(int argc, char **argv)
2639 {
2640         FILE *ways=NULL,*ways_split=NULL,*nodes=NULL,*tilesdir,*zipdir,*res;
2641         char *map=g_strdup(attrmap);
2642         int i,c,start=1,end=4,dump_coordinates=0;
2643         int keep_tmpfiles=0;
2644         int process_nodes=1, process_ways=1;
2645 #ifdef HAVE_ZLIB
2646         int compression_level=9;
2647 #else
2648         int compression_level=0;
2649 #endif
2650         int output=0;
2651         int input=0;
2652         char *result,*dbstr=NULL;
2653         FILE* input_file = stdin;
2654         struct plugins *plugins=NULL;
2655         struct attr **attrs;
2656         struct map *map_handle=NULL;
2657
2658         while (1) {
2659 #if 0
2660                 int this_option_optind = optind ? optind : 1;
2661 #endif
2662                 int option_index = 0;
2663                 static struct option long_options[] = {
2664                         {"attr-debug-level", 1, 0, 'a'},
2665                         {"binfile", 0, 0, 'b'},
2666                         {"compression-level", 1, 0, 'z'},
2667                         {"coverage", 0, 0, 'o'},
2668 #ifdef HAVE_POSTGRESQL
2669                         {"db", 1, 0, 'd'},
2670 #endif
2671                         {"dedupe-ways", 0, 0, 'w'},
2672                         {"dump", 0, 0, 'D'},
2673                         {"dump-coordinates", 0, 0, 'c'},
2674                         {"end", 1, 0, 'e'},
2675                         {"help", 0, 0, 'h'},
2676                         {"keep-tmpfiles", 0, 0, 'k'},
2677                         {"nodes-only", 0, 0, 'N'},
2678                         {"map", 1, 0, 'm'},
2679                         {"plugin", 1, 0, 'p'},
2680                         {"start", 1, 0, 's'},
2681                         {"input-file", 1, 0, 'i'},
2682                         {"ignore-unknown", 0, 0, 'n'},
2683                         {"ways-only", 0, 0, 'W'},
2684                         {0, 0, 0, 0}
2685                 };
2686                 c = getopt_long (argc, argv, "DNWa:bc"
2687 #ifdef HAVE_POSTGRESQL
2688                                               "d:"
2689 #endif
2690                                               "e:hi:knm:p:s:w", long_options, &option_index);
2691                 if (c == -1)
2692                         break;
2693                 switch (c) {
2694                 case 'D':
2695                         output=1;
2696                         break;
2697                 case 'N':
2698                         process_ways=0;
2699                         break;
2700                 case 'W':
2701                         process_nodes=0;
2702                         break;
2703                 case 'a':
2704                         attr_debug_level=atoi(optarg);
2705                         break;
2706                 case 'b':
2707                         input=1;
2708                         break;
2709                 case 'c':
2710                         dump_coordinates=1;
2711                         break;
2712 #ifdef HAVE_POSTGRESQL
2713                 case 'd':
2714                         dbstr=optarg;
2715                         break;
2716 #endif
2717                 case 'e':
2718                         end=atoi(optarg);
2719                         break;
2720                 case 'h':
2721                         usage(stdout);
2722                         break;
2723                 case 'm':
2724                         attrs=(struct attr*[]){
2725                                 &(struct attr){attr_type,{"textfile"}},
2726                                 &(struct attr){attr_data,{"bookmark.txt"}},
2727                                 NULL};
2728                         map_handle=map_new(attrs);
2729                         fprintf(stderr,"optarg=%s\n", optarg);
2730                         break;
2731                 case 'n':
2732                         fprintf(stderr,"I will IGNORE unknown types\n");
2733                         ignore_unkown=1;
2734                         break;
2735                 case 'k':
2736                         fprintf(stderr,"I will KEEP tmp files\n");
2737                         keep_tmpfiles=1;
2738                         break;
2739                 case 'o':
2740                         coverage=1;
2741                         break;
2742                 case 'p':
2743                         if (! plugins)
2744                                 plugins=plugins_new();
2745                         fprintf(stderr,"optarg=%s\n",optarg);
2746                         attrs=(struct attr*[]){&(struct attr){attr_path,{optarg}},NULL};
2747                         plugins_add_path(plugins, attrs);
2748                         break;
2749                 case 's':
2750                         start=atoi(optarg);
2751                         break;
2752                 case 'w':
2753                         dedupe_ways_hash=g_hash_table_new(NULL, NULL);
2754                         break;
2755                 case 'i':
2756                         input_file = fopen( optarg, "r" );
2757                         if ( input_file ==  NULL )
2758                         {
2759                             fprintf( stderr, "\nInput file (%s) not found\n", optarg );
2760                             exit( -1 );
2761                         }
2762                         break;
2763 #ifdef HAVE_ZLIB
2764                 case 'z':
2765                         compression_level=atoi(optarg);
2766                         break;
2767 #endif
2768                 case '?':
2769                         usage(stderr);
2770                         break;
2771                 default:
2772                         fprintf(stderr,"c=%d\n", c);
2773                 }
2774
2775         }
2776         if (optind != argc-(output == 1 ? 0:1))
2777                 usage(stderr);
2778         if (plugins)
2779                 plugins_init(plugins);
2780         result=argv[optind];
2781         build_attrmap(map);
2782 #ifdef GENERATE_INDEX
2783         build_countrytable();
2784 #endif
2785
2786
2787         if (input == 0) {
2788         if (start == 1) {
2789                 if (process_ways)
2790                         ways=fopen("ways.tmp","wb+");
2791                 if (process_nodes)
2792                         nodes=fopen("nodes.tmp","wb+");
2793                 phase=1;
2794                 fprintf(stderr,"PROGRESS: Phase 1: collecting data\n");
2795 #ifdef HAVE_POSTGRESQL
2796                 if (dbstr)
2797                         phase1_db(dbstr,ways,nodes);
2798                 else
2799 #endif
2800                 if (map_handle) {
2801                         phase1_map(map_handle,ways,nodes);
2802                         map_destroy(map_handle);
2803                 }
2804                 else
2805                         phase1(input_file,ways,nodes);
2806                 if (ways)
2807                         fclose(ways);
2808                 if (nodes)
2809                         fclose(nodes);
2810 #ifdef GENERATE_INDEX
2811                 fprintf(stderr,"PROGRESS: Phase 1: sorting countries\n");
2812                 sort_countries();
2813 #endif
2814         }
2815         if (end == 1 || dump_coordinates)
2816                 save_buffer("coords.tmp",&node_buffer);
2817         if (end == 1)
2818                 exit(0);
2819         if (start == 2)
2820                 load_buffer("coords.tmp",&node_buffer);
2821         if (start <= 2) {
2822                 if (process_ways) {
2823                         ways=fopen("ways.tmp","rb");
2824                         ways_split=fopen("ways_split.tmp","wb+");
2825                         phase=2;
2826                         fprintf(stderr,"PROGRESS: Phase 2: finding intersections\n");
2827                         phase2(ways,ways_split);
2828                         fclose(ways_split);
2829                         fclose(ways);
2830                         if(!keep_tmpfiles)
2831                                 remove("ways.tmp");
2832                 } else
2833                         fprintf(stderr,"PROGRESS: Skipping Phase 2\n");
2834         }
2835         free(node_buffer.base);
2836         node_buffer.base=NULL;
2837         node_buffer.malloced=0;
2838         node_buffer.size=0;
2839         if (end == 2)
2840                 exit(0);
2841         } else {
2842                 ways_split=fopen("ways_split.tmp","wb+");
2843                 process_binfile(stdin, ways_split);
2844                 fclose(ways_split);
2845         }
2846         if (output == 1) {
2847                 fprintf(stderr,"PROGRESS: Phase 3: dumping\n");
2848                 if (process_nodes) {
2849                         nodes=fopen("nodes.tmp","rb");
2850                         if (nodes) {
2851                                 dump(nodes);
2852                                 fclose(nodes);
2853                         }
2854                 }
2855                 if (process_ways) {
2856                         ways_split=fopen("ways_split.tmp","rb");
2857                         if (ways_split) {
2858                                 dump(ways_split);
2859                                 fclose(ways_split);
2860                         }
2861                 }
2862                 exit(0);
2863         }
2864         if (start <= 3) {
2865                 phase=3;
2866                 fprintf(stderr,"PROGRESS: Phase 3: generating tiles\n");
2867                 if (process_ways)
2868                         ways_split=fopen("ways_split.tmp","rb");
2869                 if (process_nodes)
2870                         nodes=fopen("nodes.tmp","rb");
2871                 tilesdir=fopen("tilesdir.tmp","wb+");
2872                 phase3(ways_split,nodes,tilesdir);
2873                 fclose(tilesdir);
2874                 if (nodes)
2875                         fclose(nodes);
2876                 if (ways_split)
2877                         fclose(ways_split);
2878         }
2879         if (end == 3)
2880                 exit(0);
2881         if (start <= 4) {
2882                 phase=4;
2883                 fprintf(stderr,"PROGRESS: Phase 4: assembling map\n");
2884                 if (process_ways)
2885                         ways_split=fopen("ways_split.tmp","rb");
2886                 if (process_nodes)
2887                         nodes=fopen("nodes.tmp","rb");
2888                 res=fopen(result,"wb+");
2889                 zipdir=fopen("zipdir.tmp","wb+");
2890                 phase4(ways_split,nodes,res,zipdir,compression_level);
2891                 fclose(zipdir);
2892                 fclose(res);
2893                 if (nodes)
2894                         fclose(nodes);
2895                 if (ways_split)
2896                         fclose(ways_split);
2897                 if(!keep_tmpfiles) {
2898                         remove("nodes.tmp");
2899                         remove("ways_split.tmp");
2900                         remove("tilesdir.tmp");
2901                         remove("zipdir.tmp");
2902 #ifdef GENERATE_INDEX
2903                         remove_countryfiles();
2904 #endif
2905                 }
2906         }
2907         return 0;
2908 }