Building a Visual Sitemap Generator with Python and NetworkX for Enhanced Website Analysis#

1
import requests
2
from bs4 import BeautifulSoup
3
from urllib.parse import urljoin, urlparse
4
import networkx as nx
5

6
def is_internal(url, base_url):
7
    """Checks if a URL belongs to the same domain."""
8
    return urlparse(url).netloc == urlparse(base_url).netloc
9

10
def crawl_website(start_url, max_pages=100):
11
    """Crawls a website and returns a list of (source, destination) link tuples."""
12
    visited = set()
13
    queue = [start_url]
14
    internal_links = []
15
    base_domain = urlparse(start_url).netloc
16

17
    while queue and len(visited) < max_pages:
18
        current_url = queue.pop(0)
19
        if current_url in visited:
20
            continue
21

22
        print(f"Crawling: {current_url}")
23
        visited.add(current_url)
24

25
        try:
26
            response = requests.get(current_url, timeout=5)
27
            if response.status_code != 200 or 'text/html' not in response.headers.get('Content-Type', ''):
28
                continue # Skip non-HTML or error pages
29

30
            soup = BeautifulSoup(response.text, 'html.parser')
31

32
            # Optional: Extract page title
33
            title = soup.title.string if soup.title else current_url
34

35
            for link in soup.find_all('a', href=True):
36
                href = link['href']
37
                absolute_url = urljoin(current_url, href)
38

39
                # Basic cleanup and check for internal links
40
                if '#' in absolute_url: # Ignore fragment identifiers
41
                    absolute_url = absolute_url.split('#')[0]
42

43
                if is_internal(absolute_url, start_url) and absolute_url not in visited:
44
                     if absolute_url.startswith('http') and not any(absolute_url.endswith(ext) for ext in ['.pdf', '.jpg', '.png', '.zip']): # Avoid common file types
45
                         queue.append(absolute_url)
46
                         internal_links.append((current_url, absolute_url))
47
                         # Optional: Store attributes like title or status for nodes later
48
                         # For simplicity, we collect links first, then build the graph
49
                elif is_internal(absolute_url, start_url) and absolute_url in visited:
50
                     # Link to an already visited page - record the edge
51
                     internal_links.append((current_url, absolute_url))
52

53
        except Exception as e:
54
            print(f"Error crawling {current_url}: {e}")
55
            continue
56

57
    # Need to also ensure all visited pages (nodes) are added, not just those in links
58
    all_pages = list(visited)
59
    return all_pages, internal_links
60

61
# Example usage (not runnable as a standalone script here):
62
# pages, links = crawl_website("https://www.example.com/")
63
# print(f"Found {len(pages)} pages and {len(links)} internal links.")

1
import networkx as nx
2

3
def build_sitemap_graph(pages, links):
4
    """Builds a NetworkX directed graph from pages and links."""
5
    G = nx.DiGraph()
6

7
    # Add nodes (pages)
8
    for page_url in pages:
9
        # Could add page title, status, etc. here if collected by crawler
10
        G.add_node(page_url, url=page_url)
11

12
    # Add edges (links)
13
    for source_url, dest_url in links:
14
        # Ensure both nodes exist before adding edge (important if max_pages truncates)
15
        if source_url in G.nodes and dest_url in G.nodes:
16
             G.add_edge(source_url, dest_url)
17
             # Could add edge attributes like anchor text here
18

19
    return G
20

21
# Example usage:
22
# G = build_sitemap_graph(pages, links)
23
# print(f"Graph created with {G.number_of_nodes()} nodes and {G.number_of_edges()} edges.")

1
def add_depth_attribute(graph, start_url):
2
    """Adds 'depth' attribute to nodes based on shortest path from start_url."""
3
    if start_url not in graph:
4
        print(f"Warning: Start URL {start_url} not in graph.")
5
        return
6

7
    try:
8
        # Compute shortest path lengths from the start node
9
        shortest_paths = nx.shortest_path_length(graph, source=start_url)
10

11
        # Assign depth attribute to nodes
12
        for node in graph.nodes():
13
            depth = shortest_paths.get(node, float('inf')) # Use infinity for unreachable nodes
14
            graph.nodes[node]['depth'] = depth
15
            # print(f"Node: {node}, Depth: {depth}") # For debugging
16

17
    except nx.NetworkXNoPath:
18
        print(f"Warning: Some nodes might be unreachable from {start_url}.")
19
        # Assign depth 0 only to start_url if no paths found
20
        if start_url in graph:
21
             graph.nodes[start_url]['depth'] = 0
22
        for node in graph.nodes():
23
             if 'depth' not in graph.nodes[node]:
24
                  graph.nodes[node]['depth'] = float('inf') # Explicitly mark unreachable

1
import matplotlib.pyplot as plt
2

3
def visualize_sitemap_graph(graph, layout_algo=nx.spring_layout, title="Website Visual Sitemap"):
4
    """Visualizes the NetworkX graph."""
5
    plt.figure(figsize=(12, 12))
6

7
    # Define node colors/sizes based on attributes (e.g., depth)
8
    node_colors = []
9
    node_sizes = []
10
    depths = [graph.nodes[node].get('depth', -1) for node in graph.nodes()] # -1 for nodes without depth
11

12
    # Map depth to color (e.g., darker for deeper pages)
13
    # Normalize depth for coloring or sizing
14
    max_depth = max(d for d in depths if d != float('inf') and d != -1) if depths else 0
15
    if max_depth > 0:
16
        normalized_depths = [(d / max_depth) if d != float('inf') and d != -1 else 1.1 for d in depths] # Use 1.1 for unreachable
17
        cmap = plt.cm.viridis # Color map
18
        node_colors = [cmap(nd) if nd <= 1 else 'red' for nd in normalized_depths] # Red for unreachable
19

20
        # Map depth to size (e.g., smaller for deeper pages)
21
        node_sizes = [max(50, 2000 * (1 - (d / max_depth)**0.5)) if d != float('inf') and d != -1 else 50 for d in depths] # Example size mapping
22
    else:
23
         node_colors = ['skyblue'] * graph.number_of_nodes()
24
         node_sizes = [300] * graph.number_of_nodes()
25

26

27
    # Generate layout
28
    try:
29
        pos = layout_algo(graph)
30
    except Exception as e:
31
        print(f"Could not apply layout: {e}. Falling back to spring_layout.")
32
        pos = nx.spring_layout(graph) # Fallback
33

34
    # Draw nodes
35
    nx.draw_networkx_nodes(graph, pos, node_color=node_colors, node_size=node_sizes, alpha=0.8)
36

37
    # Draw edges
38
    nx.draw_networkx_edges(graph, pos, edge_color='gray', arrows=True, alpha=0.5)
39

40
    # Optional: Draw labels (can be cluttered for large graphs)
41
    # nx.draw_networkx_labels(graph, pos, font_size=8)
42

43
    plt.title(title)
44
    plt.axis('off') # Hide axes
45
    plt.tight_layout()
46
    plt.show()
47

48
# Example usage:
49
# add_depth_attribute(G, start_url) # Assuming G is the graph and start_url is defined
50
# visualize_sitemap_graph(G)