Skip to content

Commit 283a29e

Browse files
NakanerNakaner
committed
[WIP] Add new host knoedelpresse, make data import ready for vector port
1 parent a24567a commit 283a29e

15 files changed

Lines changed: 246 additions & 46 deletions

File tree

‎ansible/group_vars/tileservers.yml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,32 @@ osm_dbname: gis
22
tileserver_script_dir: /opt/OpenRailwayMap-server-config
33
cartocss_dir: /opt/OpenRailwayMap-CartoCSS
44

5+
git:
6+
"OpenRailwayMap-CartoCSS":
7+
repo: "https://github.com/OpenRailwayMap/OpenRailwayMap-CartoCSS.git"
8+
version: "2b47c487b914eb2f255ec9d6d83aa4aff0236b42"
9+
directory: "/opt/OpenRailwayMap-CartoCSS"
10+
511
tileserver:
12+
vector: false
13+
post_import_sql_scripts:
14+
- '/opt/OpenRailwayMap-CartoCSS/sql/functions.sql'
15+
- '/opt/OpenRailwayMap-CartoCSS/sql/osm_carto_views.sql'
16+
- '/opt/OpenRailwayMap-CartoCSS/sql/get_station_importance.sql'
17+
additional_indices:
18+
- name: 'openrailwaymap_line_all_but_ab_raz_prop'
19+
where: "railway IN ('rail', 'tram', 'light_rail', 'subway', 'narrow_gauge', 'disused', 'construction')"
20+
table: planet_osm_line
21+
- name: 'openrailwaymap_line_low'
22+
where: "railway = 'rail' AND tags->'usage' IN ('main', 'branch') AND service IS NULL"
23+
table: planet_osm_line
24+
- name: 'openrailwaymap_line_med'
25+
where: "railway = 'rail' AND tags->'usage' = 'main' AND service IS NULL"
26+
table: planet_osm_line
27+
- name: 'openrailwaymap_electrification_signals'
28+
where: "railway = 'signal' AND tags ? 'railway:signal:electricity'"
29+
table: planet_osm_line
30+
631
styles:
732
- standard
833
- maxspeed
@@ -17,6 +42,12 @@ tileserver:
1742
- c.tiles.openrailwaymap.org
1843
- tiles.buegelfalte.openrailwaymap.org
1944

45+
osm2pgsql:
46+
osmium_filter: "nwr/railway r/route=train r/route=light_rail r/route=tram r/route=subway nwr/disused:railway nwr/abandoned:railway nwr/razed:railway nwr/construction:railway nwr/proposed:railway"
47+
osmium_filter_file: ""
48+
style: "/opt/OpenRailwayMap-CartoCSS/setup/openstreetmap-carto.style"
49+
tag_transform: "/opt/OpenRailwayMap-CartoCSS/setup/openstreetmap-carto.lua"
50+
2051
ufw:
2152
bans:
2253
YuLinCity-IpAddressPool-1_no_referer:

‎ansible/host_vars/knoedelpresse.openrailwaymap.org.yml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
pg_version: 17
2+
postgis_version: 3
3+
php_version: 8.4
4+
debian_version: 'trixie'
5+
osm2pgsql:
6+
output: "flex"
7+
osmium_filter: ""
8+
osmium_filter_file: "/opt/OpenRailwayMap-vector/import/osmium-tags-filter"
9+
style: "/opt/OpenRailwayMap-vector/import/openrailwaymap.lua"
10+
tag_transform: ""
11+
12+
git:
13+
"OpenRailwayMap-vector":
14+
repo: "https://github.com/OpenRailwayMap/OpenRailwayMap-CartoCSS.git"
15+
version: "7ddccefaa7e602e2e24a7ab929ee6baf35eb57e8"
16+
directory: "/opt/OpenRailwayMap-vector"
17+
18+
tileserver:
19+
post_import_sql_scripts: []
20+
additional_indices: []
21+
vector: true
22+
styles:
23+
- standard
24+
- maxspeed

‎ansible/hosts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
knallfrosch.openrailwaymap.org
2+
knoedelpresse.openrailwaymap.org
23
wal.openrailwaymap.org
34

45
[webservers]
@@ -9,6 +10,7 @@ website_servers
910

1011
[tileservers]
1112
knallfrosch.openrailwaymap.org
13+
knoedelpresse.openrailwaymap.org
1214

1315
[letsencrypt]
1416
knallfrosch.openrailwaymap.org

‎ansible/openrailwaymap/git.yml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
# SPDX-License-Identifier: MIT
2+
- hosts: all
3+
name: Clone and update Git repositories
4+
tags: git
5+
6+
tasks:
7+
- name: Install Git
8+
apt:
9+
name: git
10+
- name: Create directories for Git repositories
11+
loop: '{{ git|dict2items }}'
12+
file:
13+
path: '{{ item.value.directory }}'
14+
owner: root
15+
group: root
16+
mode: 0755
17+
state: directory
18+
- name: Clone Git repositories
19+
loop: '{{ git|dict2items }}'
20+
git:
21+
dest: '{{ item.value.directory }}'
22+
repo: '{{ item.value.repo }}'
23+
version: '{{ item.value.version }}'
24+
force: '{{ item.value.force|default(False) }}'

‎ansible/roles/tileserver/files/01_reduce_data.sh‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#! /bin/bash
2+
# SPDX-License-Identifier: GPL-3.0-or-later
3+
# Author: Hidde Wieringa <hidde@hiddewieringa.nl>
4+
# Author: Michael Reichert <osm-ml@michreichert.de>
5+
6+
set -euo pipefail
7+
8+
psql -c "delete from platforms p where not exists(select * from routes r where r.platform_ref_ids @> Array[p.osm_id]) and not exists(select * from railway_line l where st_dwithin(p.way, l.way, 20));"

‎ansible/roles/tileserver/files/02_transform_data.sh‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#! /bin/bash
2+
# SPDX-License-Identifier: GPL-3.0-or-later
3+
# Author: Hidde Wieringa <hidde@hiddewieringa.nl>
4+
# Author: Michael Reichert <osm-ml@michreichert.de>
5+
6+
set -euo pipefail
7+
8+
psql -c "update stations s set way = l.way from landuse l where ST_Within(s.way, l.way) and feature = 'yard' and GeometryType(s.way) = 'POINT' and s.osm_type = 'N';"
9+

‎ansible/roles/tileserver/files/03_create_update_functions_views.sh‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
#!/bin/bash
2+
3+
set -euo pipefail
4+
5+
echo "Post processing imported data"
6+
cd /opt/OpenRailwayMap-vector/import
7+
8+
# Functions
9+
psql -f sql/tile_functions.sql
10+
psql -f sql/api_facility_functions.sql
11+
psql -f sql/api_milestone_functions.sql
12+
13+
# YAML data
14+
psql -f sql/signal_features.sql
15+
psql -f sql/operators.sql
16+
17+
# Post processing
18+
psql -f sql/get_station_importance.sql
19+
psql -f sql/update_station_importance.sql
20+
osm2pgsql-gen \
21+
--database gis \
22+
--style openrailwaymap.lua
23+
psql -f sql/stations_clustered.sql
24+
25+
# Tile and API views on processed data
26+
psql -f sql/tile_views.sql
27+
psql -f sql/api_facility_views.sql

‎ansible/roles/tileserver/files/scripts/import.sh‎

Lines changed: 36 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -27,19 +27,48 @@ source $(dirname ${0})/config.cfg
2727

2828
echo "Started processing at $(date)"
2929

30-
echo "[1/3] Downloading planet file"
31-
wget --progress=dot:giga --no-clobber -O $PLANET_FILE https://planet.openstreetmap.org/pbf/planet-latest.osm.pbf
30+
echo "[1/4] Downloading planet file"
31+
wget --progress=dot:giga --no-clobber -O $PLANET_FILE --continue https://planet.openstreetmap.org/pbf/planet-latest.osm.pbf || true
32+
RET_CODE=$?
33+
if [ $RET_CODE -eq 1 ]; then
34+
PLANET_SIZE=$(stat --printf="%s" "$PLANET_FILE")
35+
if [ "$PLANET_SIZE" -lt 91279000000 ]; then
36+
echo "ERROR: Planet download failed with return code $RET_CODE and found file is too small."
37+
exit 1
38+
fi
39+
elif [ $RET_CODE -gt 0 ]; then
40+
echo "ERROR: Planet download failed with return code $RET_CODE and found file is too small."
41+
exit 1
42+
fi
3243

33-
echo "[2/3] Filtering data extract"
34-
$OSMIUM tags-filter -o $PLANET_FILTERED $PLANET_FILE $OSMIUM_FILTER_EXPR
44+
echo "[2/4] Filtering data extract"
45+
FILTER_ARG=""
46+
if [[ -n "OSMIUM_FILTER_EXPR_FILE" ]]; then
47+
FILTER_ARG="-e $OSMIUM_FILTER_EXPR_FILE"
48+
fi
49+
$OSMIUM tags-filter $FILTER_ARG -o $PLANET_FILTERED $PLANET_FILE $OSMIUM_FILTER_EXPR
3550

36-
echo "[3/3] Import data into database"
51+
echo "[3/4] Import data into database"
3752
if [[ -v "OSM2PGSQL_FLATNODES" ]]; then
38-
FLATNODES_OPTION="--flat-node $FLATNODES_FILE"
53+
FLATNODES_OPTION="--flat-node $OSM2PGSQL_FLATNODES"
3954
else
4055
FLATNODES_OPTION=""
4156
fi
42-
$OSM2PGSQL -d $DATABASE_NAME --merc --multi-geometry --hstore --style $OSM2PGSQL_STYLE --tag-transform $OSM2PGSQL_LUA --slim $FLATNODES_OPTION $PLANET_FILTERED
57+
if [[ -n "OSM2PGSQL_TAG_TRANSFORM" ]]; then
58+
TAG_TRANSFORM_OPTION="--tag-transform $OSM2PGSQL_TAG_TRANSFORM"
59+
else
60+
TAG_TRANSFORM_OPTION=""
61+
fi
62+
if [ "$OSM2PGSQL_OUTPUT" = "pgsql" ]; then
63+
EXTRA_OPTS="--merc --hstore"
64+
else
65+
EXTRA_OPTS=""
66+
fi
67+
68+
$OSM2PGSQL --create -d $DATABASE_NAME --output $OSM2PGSQL_OUTPUT $EXTRA_OPTS --multi-geometry --style $OSM2PGSQL_STYLE $TAG_TRANSFORM_OPTION --slim $FLATNODES_OPTION $PLANET_FILTERED
69+
70+
echo "[4/4] Running additional update scripts in /opt/OpenRailwayMap-server-config/post-update.d/"
71+
run-parts --exit-on-error -v /opt/OpenRailwayMap-server-config/post-import.d/
4372

4473
REPLICATION_TIMESTAMP=$($OSMIUM fileinfo -g header.option.osmosis_replication_timestamp $PLANET_FILE)
4574
echo "replication timestamp is $REPLICATION_TIMESTAMP"

‎ansible/roles/tileserver/files/scripts/update_osm.sh‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,13 +44,23 @@ function apply_diff_database {
4444
else
4545
FLATNODES_OPTION=""
4646
fi
47+
if [[ -n "OSM2PGSQL_TAG_TRANSFORM" ]]; then
48+
TAG_TRANSFORM_OPTION="--tag-transform $OSM2PGSQL_TAG_TRANSFORM"
49+
else
50+
TAG_TRANSFORM_OPTION=""
51+
fi
52+
if [ "$OSM2PGSQL_OUTPUT" = "pgsql" ]; then
53+
EXTRA_OPTS="--merc --hstore"
54+
else
55+
EXTRA_OPTS=""
56+
fi
4757
if [[ -v "OSM2PGSQL_NUMBER_PROCESSES" ]]; then
4858
NUMBER_PROCESSES_OPTION="--number-processes $OSM2PGSQL_NUMBER_PROCESSES"
4959
else
5060
NUMBER_PROCESSES_OPTION=""
5161
fi
5262
OSM2PGSQL_RETURNCODE=0
53-
$OSM2PGSQL --append -d $DATABASE_NAME --merc --multi-geometry --hstore --style $OSM2PGSQL_STYLE --tag-transform $OSM2PGSQL_LUA --slim $FLATNODES_OPTION $NUMBER_PROCESSES_OPTION $DERIVED_DIFF || OSM2PGSQL_RETURNCODE=$?
63+
$OSM2PGSQL --append -d $DATABASE_NAME --output $OSM2PGSQL_OUTPUT $EXTRA_OPTS --multi-geometry --style $OSM2PGSQL_STYLE $TAG_TRANSFORM_OPTION --slim $FLATNODES_OPTION $NUMBER_PROCESSES_OPTION $DERIVED_DIFF || OSM2PGSQL_RETURNCODE=$?
5464

5565
if [ "$OSM2PGSQL_RETURNCODE" -gt 0 ] ; then
5666
echo "Osm2pgsql failed with return code $OSM2PGSQL_RETURNCODE, storing diff file in $LAST_DERIVED_DIFF"
@@ -105,7 +115,11 @@ echo "filtering planet"
105115
if [ -f "$PLANET_FILTERED" ]; then
106116
mv $PLANET_FILTERED $PLANET_FILTERED_OLD
107117
fi
108-
$OSMIUM tags-filter -o $PLANET_FILTERED $PLANET_FILE $OSMIUM_FILTER_EXPR
118+
FILTER_ARG=""
119+
if [[ -n "OSMIUM_FILTER_EXPR_FILE" ]]; then
120+
FILTER_ARG="-e $OSMIUM_FILTER_EXPR_FILE"
121+
fi
122+
$OSMIUM tags-filter $FILTER_ARG -o $PLANET_FILTERED $PLANET_FILE $OSMIUM_FILTER_EXPR
109123

110124
if [ "$TILE_RENDERING" = 1 ]; then
111125
apply_diff_database

‎ansible/roles/tileserver/tasks/main.yml‎

Lines changed: 4 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,10 @@
9797
apt:
9898
name: wget,gdal-bin,nodejs,npm,libwww-perl,mapnik-utils,bc,git,osmium-tool,python3-pyosmium,pyosmium
9999

100+
- name: Build Lua configuration for import of vectortile port
101+
when: tileserver.vector
102+
include_tasks: vector.yml
103+
100104
- name: Create directories for scripts
101105
file:
102106
path: '{{ item }}'
@@ -225,21 +229,6 @@
225229
apt:
226230
name: [fonts-unifont]
227231

228-
- name: Create directory for CartoCSS map style repository
229-
file:
230-
path: '{{ cartocss_dir }}'
231-
owner: osmimport
232-
group: osmimport
233-
state: directory
234-
mode: '0755'
235-
236-
- name: Clone CartoCSS map style repository
237-
become_user: osmimport
238-
git:
239-
dest: '{{ cartocss_dir }}'
240-
repo: https://github.com/OpenRailwayMap/OpenRailwayMap-CartoCSS.git
241-
version: 2b47c487b914eb2f255ec9d6d83aa4aff0236b42
242-
243232
- name: Check if osm_import runs
244233
include_tasks: osm_import_active.yml
245234

0 commit comments

Comments
 (0)