diff options
| -rw-r--r-- | .gitlab-ci.yml | 2 | ||||
| -rw-r--r-- | Makefile | 55 |
2 files changed, 29 insertions, 28 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8fb3cd6..7a1bd25 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -147,8 +147,6 @@ conflate: script: - yarn install - mkdir -p dist - - touch data/victoria-addr.osm.pbf - - touch --no-create data/victoria-addr.osm.geojson - make dist/conflate when: manual artifacts: @@ -10,18 +10,18 @@ data/VICMAP_PROPERTY.zip: mkdir -p data wget --no-verbose --directory-prefix=data https://www.alantgeo.com.au/share/VICMAP_PROPERTY.zip -data/vicmap/ll_gda94/sde_shape/whole/VIC/VMADD/layer/address.shp: data/VICMAP_ADDRESS.zip +data/vicmap/ll_gda94/sde_shape/whole/VIC/VMADD/layer/address.shp: | data/VICMAP_ADDRESS.zip mkdir -p data/vicmap unzip -d data/vicmap -n $< # update mtime so that Make doesn't see it as outdated touch --no-create $@ -data/vicmap/ll_gda94/sde_shape/whole/VIC/VMPROP/layer/property_view.shp: data/VICMAP_PROPERTY.zip +data/vicmap/ll_gda94/sde_shape/whole/VIC/VMPROP/layer/property_view.shp: | data/VICMAP_PROPERTY.zip mkdir -p data/vicmap unzip -d data/vicmap -n $< touch --no-create $@ -data/vicmap-property.fgb: data/vicmap/ll_gda94/sde_shape/whole/VIC/VMPROP/layer/property_view.shp +data/vicmap-property.fgb: | data/vicmap/ll_gda94/sde_shape/whole/VIC/VMPROP/layer/property_view.shp ogr2ogr -f FlatGeobuf -nlt PROMOTE_TO_MULTI $@ $< data/vicmap.geojson: @@ -35,58 +35,61 @@ vicmapExtract: cleanDist: rm -rf dist -dist/vicmap-osm.geojson: data/vicmap.geojson +dist/vicmap-osm.geojson: | data/vicmap.geojson mkdir -p dist ./bin/vicmap2osm.js $< $@ wc -l $@ -dist/vicmap-osm.mbtiles: dist/vicmap-osm.geojson +dist/vicmap-osm.mbtiles: | dist/vicmap-osm.geojson tippecanoe --force -o $@ --minimum-zoom=12 --maximum-zoom=12 --no-feature-limit --no-tile-size-limit --no-tile-stats --read-parallel $< -dist/vicmap-osm-uniq.geojson: dist/vicmap-osm.geojson +dist/vicmap-osm-uniq.geojson: | dist/vicmap-osm.geojson mkdir -p debug/reduceDuplicates node --max_old_space_size=4096 ./bin/reduceDuplicates.js --debug $< $@ -dist/vicmap-osm-uniq-flats.geojson: dist/vicmap-osm-uniq.geojson +dist/vicmap-osm-uniq-flats.geojson: | dist/vicmap-osm-uniq.geojson mkdir -p debug/reduceOverlap node --max_old_space_size=4096 ./bin/reduceOverlap.js --debug $< $@ -dist/vicmap-osm-uniq-flats-withinrange.geojson: dist/vicmap-osm-uniq-flats.geojson +dist/vicmap-osm-uniq-flats-withinrange.geojson: | dist/vicmap-osm-uniq-flats.geojson mkdir -p debug/reduceRangeDuplicates node --max_old_space_size=4096 ./bin/reduceRangeDuplicates.js --debug $< $@ -loadPgOSM: dist/vicmap-osm.geojson +dist/canidates.geojson: | dist/vicmap-osm-uniq-flats-withinrange.geojson + cp $< $@ + +loadPgOSM: | dist/vicmap-osm.geojson ogr2ogr -f PostgreSQL PG: $< -lco UNLOGGED=YES -nln vm_osm -data/vicmap.fgb: data/vicmap/ll_gda94/sde_shape/whole/VIC/VMADD/layer/address.shp +data/vicmap.fgb: | data/vicmap/ll_gda94/sde_shape/whole/VIC/VMADD/layer/address.shp ogr2ogr -f FlatGeobuf $@ $< -dist/vicmap-osm.fgb: dist/vicmap-osm.geojson +dist/vicmap-osm.fgb: | dist/vicmap-osm.geojson ogr2ogr -f FlatGeobuf $@ $< # useful for development to be able to query a database -loadPgAdd: data/vicmap/ll_gda94/sde_shape/whole/VIC/VMADD/layer/address.shp +loadPgAdd: | data/vicmap/ll_gda94/sde_shape/whole/VIC/VMADD/layer/address.shp ogr2ogr -f PostgreSQL PG: $< -lco UNLOGGED=YES -nln vmadd # index all columns for faster queries during development psql -f src/createIndexQuery.sql --tuples-only | psql -loadPgProp: data/vicmap/ll_gda94/sde_shape/whole/VIC/VMPROP/layer/property_view.shp +loadPgProp: | data/vicmap/ll_gda94/sde_shape/whole/VIC/VMPROP/layer/property_view.shp ogr2ogr -f PostgreSQL PG: $< -lco UNLOGGED=YES -nln vmprop -nlt MULTIPOLYGON data/victoria.osm.pbf: wget --no-verbose --directory-prefix=data http://download.openstreetmap.fr/extracts/oceania/australia/victoria.osm.pbf # addr:suburb, addr:postcode alone without a housenumber or being an interpolation way aren't of much use for comparisons -data/victoria-addr.osm.pbf: data/victoria.osm.pbf +data/victoria-addr.osm.pbf: | data/victoria.osm.pbf osmium tags-filter --output=$@ --overwrite $< addr:housenumber addr:interpolation -data/victoria-addr.osm.geojson: data/victoria-addr.osm.pbf +data/victoria-addr.osm.geojson: | data/victoria-addr.osm.pbf osmium export --config=config/osmium-export-config.json --output-format=geojsonseq --format-option=print_record_separator=false --output=$@ --overwrite $< -data/victoria-addr.osm.fgb: data/victoria-addr.osm.geojson +data/victoria-addr.osm.fgb: | data/victoria-addr.osm.geojson ogr2ogr -f FlatGeobuf -nlt PROMOTE_TO_MULTI -skipfailures -mapFieldType Integer64List=String $@ $< -data/victoria-addr.osm.centroids.fgb: data/victoria-addr.osm.fgb +data/victoria-addr.osm.centroids.fgb: | data/victoria-addr.osm.fgb qgis_process run native:centroids -- INPUT='$<|layername=victoria-addr.osm|option:VERIFY_BUFFERS=NO' OUTPUT=$@ data/asgs.zip: @@ -98,15 +101,15 @@ loadMB: data/mb.geojson: ogr2ogr -f GeoJSON -where 'STATE_CODE_2016 = 2' $@ /vsizip/data/asgs.zip/ASGS\ 2016\ Volume\ 1.gpkg -lco WRITE_BBOX=YES -lco COORDINATE_PRECISION=7 -lco RFC7946=YES -lco WRITE_NAME=NO -lco ID_FIELD=MB_CODE_2016 -nln mb -select 'MB_CODE_2016' MB_2016_AUST -data/mb.fgb: data/mb.geojson +data/mb.fgb: | data/mb.geojson ogr2ogr -f FlatGeobuf $@ $< # extract roads from OSM -data/victoria-roads.osm.pbf: data/victoria.osm.pbf +data/victoria-roads.osm.pbf: | data/victoria.osm.pbf osmium tags-filter --remove-tags --output=$@ $< w/highway=motorway,trunk,primary,secondary,tertiary,unclassified,residential,living_street,road # extract road lines into geojson -data/victoria-roads.geojson: data/victoria-roads.osm.pbf +data/victoria-roads.geojson: | data/victoria-roads.osm.pbf osmium export --geometry-types=linestring --output-format=geojsonseq --format-option=print_record_separator=false --output $@ $< data/victoria-boundary.geojson: @@ -114,23 +117,23 @@ data/victoria-boundary.geojson: cat $@ >> data/victoria-roads.geojson # then convert to fgb -data/victoria-roads.fgb: data/victoria-roads.geojson data/victoria-boundary.geojson +data/victoria-roads.fgb: | data/victoria-roads.geojson data/victoria-boundary.geojson ogr2ogr -f FlatGeobuf -explodecollections -nlt MULTILINESTRING $@ $< # construct block polygons based on OSM roads -data/blocks.fgb: data/victoria-roads.fgb +data/blocks.fgb: | data/victoria-roads.fgb qgis_process run native:polygonize -- INPUT=$< KEEP_FIELDS=FALSE OUTPUT=$@ # count OSM addresses by block, those with no OSM addresses we can import all the candidate addresses without conflation issues -dist/blocksByOSMAddr.fgb: data/victoria-addr.osm.centroids.fgb data/blocks.fgb +dist/blocksByOSMAddr.fgb: | data/victoria-addr.osm.centroids.fgb data/blocks.fgb qgis_process run native:countpointsinpolygon -- POINTS=$< POLYGONS='data/blocks.fgb|layername=blocks' FIELD=NUMPOINTS OUTPUT=$@ -dist/blocksByOSMAddr.geojson: dist/blocksByOSMAddr.fgb +dist/blocksByOSMAddr.geojson: | dist/blocksByOSMAddr.fgb ogr2ogr -f GeoJSONSeq $@ $< -summariseBlocksByOSMAddr: dist/blocksByOSMAddr.geojson +summariseBlocksByOSMAddr: | dist/blocksByOSMAddr.geojson ogrinfo -dialect sqlite -sql 'select count(*), NUMPOINTS = 0 from blocksByOSMAddr group by (NUMPOINTS = 0)' $< # conflate processed vicmap data with osm data -dist/conflate: dist/vicmap-osm-uniq-flats.geojson data/victoria-addr.osm.geojson dist/blocksByOSMAddr.geojson +dist/conflate: | dist/vicmap-osm-uniq-flats.geojson data/victoria-addr.osm.geojson dist/blocksByOSMAddr.geojson ./bin/conflate.js $^ $@ |