Merge pull request 'Update and fix plz' (#31) from dev into main
Reviewed-on: #31
This commit is contained in:
commit
1e5b198f91
|
@ -42,5 +42,5 @@ python manage.py getplzindex --force
|
||||||
To get started with development, see [DEVELOPMENT.md](DEVELOPMENT.md)
|
To get started with development, see [DEVELOPMENT.md](DEVELOPMENT.md)
|
||||||
|
|
||||||
## Open Source Data
|
## Open Source Data
|
||||||
For calculating distances between zip codes, the `PLZ_Verzeichnis` is used.
|
For calculating distances between zip codes, the `Official index of cities and towns including postal codes and perimeter ` is used.
|
||||||
Source: https://opendata.swiss/de/dataset/plz_verzeichnis
|
Source: https://opendata.swiss/en/dataset/amtliches-ortschaftenverzeichnis-mit-postleitzahl-und-perimeter
|
|
@ -3,7 +3,6 @@ python manage.py migrate
|
||||||
python manage.py loco
|
python manage.py loco
|
||||||
python manage.py makemessages -l en -l de
|
python manage.py makemessages -l en -l de
|
||||||
python manage.py compilemessages -f
|
python manage.py compilemessages -f
|
||||||
python manage.py getplzindex
|
|
||||||
|
|
||||||
nginx
|
nginx
|
||||||
|
|
||||||
|
|
|
@ -1,78 +0,0 @@
|
||||||
import json
|
|
||||||
import os
|
|
||||||
from urllib import request
|
|
||||||
|
|
||||||
import pandas as pd
|
|
||||||
from django.core.management.base import BaseCommand
|
|
||||||
from pflaenzli_django.settings import BASE_DIR
|
|
||||||
from pflaenzli.models import Plz
|
|
||||||
|
|
||||||
|
|
||||||
class Command(BaseCommand):
|
|
||||||
help = 'Get the zip code index from post and compile it to a dataframe pickle'
|
|
||||||
|
|
||||||
def add_arguments(self, parser):
|
|
||||||
parser.add_argument("--force", action="store_true", required=False)
|
|
||||||
|
|
||||||
def handle(self, *args, **options):
|
|
||||||
self.parse_data(*self.download_geojson(api='v2', data='v2', force=options["force"]), force=options["force"])
|
|
||||||
|
|
||||||
def download_geojson(self, api, data, force=False):
|
|
||||||
file = f'plz_verzeichnis_{data}.json'
|
|
||||||
if os.path.exists(file) and not force:
|
|
||||||
self.stdout.write('File already downloaded.')
|
|
||||||
self.stdout.write(self.style.SUCCESS(
|
|
||||||
'Skipping...\n'))
|
|
||||||
exists = True
|
|
||||||
else:
|
|
||||||
self.stdout.write('Downloading geojson...')
|
|
||||||
url = f'https://swisspost.opendatasoft.com/api/{api}/catalog/datasets/plz_verzeichnis_{data}/exports/geojson'
|
|
||||||
request.urlretrieve(url, file)
|
|
||||||
self.stdout.write(self.style.SUCCESS('Done!\n'))
|
|
||||||
exists = False
|
|
||||||
|
|
||||||
return file, exists
|
|
||||||
|
|
||||||
def parse_data(self, file, exists, force=False):
|
|
||||||
if exists and not force:
|
|
||||||
self.stdout.write(self.style.WARNING(
|
|
||||||
'Nothing was done, if you want to redownload the PLZ index, use the --force option.\n'))
|
|
||||||
return
|
|
||||||
|
|
||||||
self.stdout.write('Opening file...')
|
|
||||||
# Load the GeoJSON data for the zip codes
|
|
||||||
with open(file, encoding='UTF-8') as f:
|
|
||||||
full_data = json.load(f)
|
|
||||||
self.stdout.write(self.style.SUCCESS('Done!\n'))
|
|
||||||
|
|
||||||
self.stdout.write('Deleting existing data...')
|
|
||||||
Plz.objects.all().delete()
|
|
||||||
self.stdout.write(self.style.SUCCESS('Done!\n'))
|
|
||||||
|
|
||||||
self.stdout.write('Parsing file and add new data...')
|
|
||||||
|
|
||||||
for plz_entry in full_data['features']:
|
|
||||||
plz_entry = plz_entry['properties']
|
|
||||||
try:
|
|
||||||
plz = plz_entry['postleitzahl']
|
|
||||||
except (AttributeError, TypeError, ValueError):
|
|
||||||
continue
|
|
||||||
|
|
||||||
try:
|
|
||||||
lat = plz_entry['geo_point_2d']['lat']
|
|
||||||
lon = plz_entry['geo_point_2d']['lon']
|
|
||||||
except (AttributeError, TypeError):
|
|
||||||
continue
|
|
||||||
|
|
||||||
if plz is None or lat is None or lon is None:
|
|
||||||
continue
|
|
||||||
|
|
||||||
try:
|
|
||||||
name = plz_entry['ortbez27']
|
|
||||||
except (KeyError, AttributeError, TypeError):
|
|
||||||
name = None
|
|
||||||
|
|
||||||
plz, _ = Plz.objects.get_or_create(plz=int(plz), lat=lat, lon=lon, name=name)
|
|
||||||
|
|
||||||
self.stdout.write(self.style.SUCCESS('Wrote PLZ data to the databse successfully\n'))
|
|
||||||
self.stdout.write(self.style.SUCCESS('Done!'))
|
|
|
@ -0,0 +1,24 @@
|
||||||
|
# Generated by Django 4.2.10 on 2024-02-10 11:16
|
||||||
|
|
||||||
|
from django.db import migrations, models
|
||||||
|
from pflaenzli.utils.load_plz import load_plz
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
replaces = [("pflaenzli", "0007_alter_plz_name"), ("pflaenzli", "0008_load_plz")]
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
("pflaenzli", "0006_plz"),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name="plz",
|
||||||
|
name="name",
|
||||||
|
field=models.CharField(max_length=40),
|
||||||
|
),
|
||||||
|
migrations.RunPython(
|
||||||
|
code=load_plz,
|
||||||
|
),
|
||||||
|
]
|
|
@ -18,7 +18,7 @@ class Plz(models.Model):
|
||||||
plz = models.IntegerField(verbose_name='PLZ')
|
plz = models.IntegerField(verbose_name='PLZ')
|
||||||
lat = models.DecimalField(max_digits=8, decimal_places=6)
|
lat = models.DecimalField(max_digits=8, decimal_places=6)
|
||||||
lon = models.DecimalField(max_digits=8, decimal_places=6)
|
lon = models.DecimalField(max_digits=8, decimal_places=6)
|
||||||
name = models.CharField(max_length=27)
|
name = models.CharField(max_length=40)
|
||||||
|
|
||||||
|
|
||||||
class Offer(models.Model):
|
class Offer(models.Model):
|
||||||
|
|
|
@ -20,11 +20,18 @@ def filter_by_distance(qs, filter_zipcode, max_dist):
|
||||||
|
|
||||||
try:
|
try:
|
||||||
filter_plz = Plz.objects.get(plz=filter_zipcode)
|
filter_plz = Plz.objects.get(plz=filter_zipcode)
|
||||||
|
except Plz.MultipleObjectsReturned:
|
||||||
|
filter_plz = Plz.objects.filter(plz=filter_zipcode)[0]
|
||||||
except Plz.DoesNotExist:
|
except Plz.DoesNotExist:
|
||||||
return filtered_offers
|
return filtered_offers
|
||||||
|
|
||||||
for offer in qs:
|
for offer in qs:
|
||||||
offer_plz = Plz.objects.get(plz=offer.zipcode)
|
try:
|
||||||
|
offer_plz = Plz.objects.get(plz=offer.zipcode)
|
||||||
|
except Plz.MultipleObjectsReturned:
|
||||||
|
offer_plz = Plz.objects.filter(plz=offer.zipcode)[0]
|
||||||
|
except Plz.DoesNotExist:
|
||||||
|
offer_plz = None
|
||||||
|
|
||||||
d = calculate_distance(offer_plz, filter_plz)
|
d = calculate_distance(offer_plz, filter_plz)
|
||||||
if d is not None and d <= max_dist:
|
if d is not None and d <= max_dist:
|
||||||
|
|
65
pflaenzli/pflaenzli/utils/load_plz.py
Normal file
65
pflaenzli/pflaenzli/utils/load_plz.py
Normal file
|
@ -0,0 +1,65 @@
|
||||||
|
import os
|
||||||
|
from urllib import request
|
||||||
|
import hashlib
|
||||||
|
import shutil
|
||||||
|
import pandas as pd
|
||||||
|
|
||||||
|
from pflaenzli.models import Plz
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
FILENAME = "AMTOVZ_CSV_WGS84"
|
||||||
|
FILEPATH = os.path.join(FILENAME, f"{FILENAME}.csv")
|
||||||
|
|
||||||
|
|
||||||
|
def add_arguments(parser):
|
||||||
|
parser.add_argument("--force", action="store_true", required=False)
|
||||||
|
|
||||||
|
def load_plz(apps, schema_editor, force = True):
|
||||||
|
get_index()
|
||||||
|
if force or not hash_matches():
|
||||||
|
create_objects()
|
||||||
|
|
||||||
|
|
||||||
|
def hash_file(filename):
|
||||||
|
h = hashlib.sha1()
|
||||||
|
|
||||||
|
with open(filename,'rb') as file:
|
||||||
|
chunk = 0
|
||||||
|
while chunk != b'':
|
||||||
|
# read only 1024 bytes at a time
|
||||||
|
chunk = file.read(1024)
|
||||||
|
h.update(chunk)
|
||||||
|
|
||||||
|
return h.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def get_index(url="https://data.geo.admin.ch/ch.swisstopo-vd.ortschaftenverzeichnis_plz/ortschaftenverzeichnis_plz/ortschaftenverzeichnis_plz_4326.csv.zip"):
|
||||||
|
request.urlretrieve(url, 'index.zip')
|
||||||
|
shutil.unpack_archive('index.zip')
|
||||||
|
|
||||||
|
|
||||||
|
def hash_matches(file=FILEPATH, hashfile="hash"):
|
||||||
|
if os.path.exists(hashfile):
|
||||||
|
with open(hashfile, 'r+') as f:
|
||||||
|
old_hash = f.read()
|
||||||
|
new_hash = hash_file(file)
|
||||||
|
|
||||||
|
if old_hash == new_hash:
|
||||||
|
return True
|
||||||
|
|
||||||
|
f.truncate(0)
|
||||||
|
f.write(new_hash)
|
||||||
|
else:
|
||||||
|
with open(hashfile, 'w') as f:
|
||||||
|
f.write(hash_file(file))
|
||||||
|
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def create_objects(file=FILEPATH):
|
||||||
|
df = pd.read_csv(file, sep=";")
|
||||||
|
objects = [Plz(plz=row['PLZ'], lat=row['N'], lon=row['E'], name=row['Ortschaftsname']) for index, row in df.iterrows()]
|
||||||
|
|
||||||
|
Plz.objects.all().delete()
|
||||||
|
Plz.objects.bulk_create(objects)
|
|
@ -1,14 +1,14 @@
|
||||||
crispy-bootstrap5==0.7
|
crispy-bootstrap5==2023.10
|
||||||
Django==4.1.11
|
Django==4.2.10
|
||||||
djangoloco==1.0
|
djangoloco==1.0
|
||||||
django-bootstrap5==23.3
|
django-bootstrap5==23.4
|
||||||
django-crispy-forms==2.0
|
django-crispy-forms==2.1
|
||||||
django-jquery==3.1.0
|
django-jquery==3.1.0
|
||||||
django-friendly-captcha==0.1.8
|
django-friendly-captcha==0.1.8
|
||||||
geopy==2.4.0
|
geopy==2.4.1
|
||||||
gunicorn==21.2.0
|
gunicorn==21.2.0
|
||||||
fontawesomefree==6.4.2
|
fontawesomefree==6.5.1
|
||||||
pandas==2.1.0
|
pandas==2.2.0
|
||||||
Pillow==10.0.1
|
Pillow==10.2.0
|
||||||
psycopg2-binary==2.9.7
|
psycopg2-binary==2.9.9
|
||||||
python-dotenv==1.0.0
|
python-dotenv==1.0.1
|
||||||
|
|
Loading…
Reference in a new issue