Recently while updating business information in my area, I noticed that outdated POIs would often include a defunct website. When attempting to follow the links in OSM, I was just presented with an error. That gave me the idea that I could proactively identify POIs that may need validated in person.
After some tinkering, I came up with a python script that will query OSM data for nodes with websites, then iterate over the websites to see which return errors when connecting. I started with logging these to a text file, but then realized that I could export them all to a kmz file and import them into Organic Maps. Now when I’m out and about I can launch Organic Maps to find areas that I should validate.
When you select a pin, it’ll give you more details on the failure.
While testing I noticed that there are also many sites getting errors indicating that the specific page isn’t found or that the page can be found at a new location. I believe that these can be validated without surveying, so I set up a separate mode called “armchair” mode that highlight these errors.
Setup
NOTE: This will initiate connections from your machine to whatever websites exist in OSM
To use the script, copy the below file and save it as “broken-websites.py”. Before running I believe you’ll need to install the required packages by running the below commands in your terminal:
pip3 install simplekml
pip3 install overpy
pip3 install geopy
From there you can change that latitude, longitude, and radius. Keep in mind that a larger radius takes exponentially longer. You can commands like:
# List all websites that return a 5XX error to a text file
python3 ./broken-websites.py survey txt
# Generate a kmz of all websites that redirect or return 404
python3 ./broken-websites.py airchair kmz
# Generate both list types to both file types
python3 ./broken-websites.py both both
Script:
import overpy
from geopy.distance import geodesic
import requests
import datetime
import simplekml
import sys
latitude = 52.377956
longitude = 4.897070
radius = .5 # Default radius in kilometers
def check_website(url):
try:
response = requests.head(url, timeout=10)
return response.status_code, response.reason
except Exception as e:
return None, str(e)
def get_amenities():
api = overpy.Overpass()
# Define bounding box
lat_min = latitude - (radius * 0.009)
lat_max = latitude + (radius * 0.009)
lon_min = longitude - (radius * 0.009)
lon_max = longitude + (radius * 0.009)
query_str = f"""
node["website"]
({lat_min},{lon_min},{lat_max},{lon_max});
out;
"""
result = api.query(query_str)
amenities = []
for node in result.nodes:
name = node.tags.get("name", "Unknown")
amenity_type = node.tags.get("amenity", "Unknown")
lat = float(node.lat)
lon = float(node.lon)
website = node.tags.get("website", "N/A")
distance = geodesic((latitude, longitude), (lat, lon)).kilometers
osm_link = f"https://www.openstreetmap.org/node/{node.id}"
if distance <= radius:
amenities.append({"name": name, "amenity_type": amenity_type, "latitude": lat, "longitude": lon, "website": website, "distance": distance, "osm_link": osm_link})
return amenities
def save_to_kmz(amenities_with_broken_websites, mode, timestamp):
kml = simplekml.Kml()
icon_url = 'https://upload.wikimedia.org/wikipedia/commons/e/ec/Red_dot.svg' # Privacy-friendly icon URL
for amenity in amenities_with_broken_websites:
description = (f"<p>Website: <a href='{amenity['website']}' target='_blank'>{amenity['website']}</a></p>"
f"<p>{amenity['error_message']}</p>"
f"<p><a href='{amenity['osm_link']}' target='_blank'>Link to OSM</a></p>")
placemark = kml.newpoint(name=amenity['name'],
description=description,
coords=[(amenity['longitude'], amenity['latitude'])])
placemark.style.iconstyle.icon.href = icon_url
file_name = (f"Broken_Websites_{mode}_Radius_{radius}km_Lat_{latitude}_Long_{longitude}_{timestamp}.kmz")
kml.savekmz(file_name)
print(f"KMZ file saved as: {file_name}")
def save_to_txt(amenities_with_broken_websites, mode, timestamp):
file_name = (f"Broken_Websites_{mode}_Radius_{radius}km_Lat_{latitude}_Long_{longitude}_{timestamp}.txt")
with open(file_name, 'w') as f:
for amenity in amenities_with_broken_websites:
f.write(f"Name: {amenity['name']}\n")
f.write(f"Amenity Type: {amenity['amenity_type']}\n")
f.write(f"Latitude: {amenity['latitude']}\n")
f.write(f"Longitude: {amenity['longitude']}\n")
f.write(f"Website: {amenity['website']}\n")
f.write(f"Distance: {amenity['distance']:.2f} km\n")
f.write(f"Error: {amenity['error_message']}\n")
f.write(f"OpenStreetMap: {amenity['osm_link']}\n")
f.write("\n")
print(f"Text file saved as: {file_name}")
def filter_amenities(amenities):
filtered_amenities = {"survey": [], "armchair": []}
for amenity in amenities:
status_code, error_message = check_website(amenity['website'])
if not status_code or 500 <= status_code < 600:
amenity['error_message'] = f"Status Code: {status_code}, Error: {error_message}"
filtered_amenities["survey"].append(amenity)
if status_code and (300 <= status_code < 400 or status_code == 404):
amenity['error_message'] = f"Status Code: {status_code}, Error: {error_message}"
filtered_amenities["armchair"].append(amenity)
return filtered_amenities
def main():
if len(sys.argv) != 3:
print("Usage: python script.py <mode> <output>")
print("Mode should be one of: survey, armchair, both")
print("Output should be one of: txt, kmz, both")
return
mode = sys.argv[1]
output = sys.argv[2]
if mode not in ["survey", "armchair", "both"]:
print("Invalid mode. Mode should be one of: survey, armchair, both")
return
if output not in ["txt", "kmz", "both"]:
print("Invalid output. Output should be one of: txt, kmz, both")
return
print(f"Searching for amenities with websites within {radius} km...")
amenities = get_amenities()
if amenities:
print("\nChecking websites...")
filtered_amenities = filter_amenities(amenities)
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
if mode in ["survey", "both"]:
if filtered_amenities["survey"]:
# print(f"Found {len(filtered_amenities["survey"])} broken websites")
if output in ["kmz", "both"]:
save_to_kmz(filtered_amenities["survey"], "survey", timestamp)
if output in ["txt", "both"]:
save_to_txt(filtered_amenities["survey"], "survey", timestamp)
else:
print("No amenities found with broken websites in survey mode.")
if mode in ["armchair", "both"]:
if filtered_amenities["armchair"]:
# print(f"Found {len(filtered_amenities["armchair"])} broken websites")
if output in ["kmz", "both"]:
save_to_kmz(filtered_amenities["armchair"], "armchair", timestamp)
if output in ["txt", "both"]:
save_to_txt(filtered_amenities["armchair"], "armchair", timestamp)
else:
print("No amenities found with broken websites in armchair mode.")
else:
print("No amenities found within the specified radius or with specified website.")
if __name__ == "__main__":
main()
OpenStreetMap
© OpenStreetMap Contributors. Screenshot from JOSM with Bing
© OpenStreetMap Contributors. Screenshot from JOSM with OSM Carto
.jpg)
.jpg){kind=small}
