import os import requests import networkx as nx import pandas as pd # we'll need to rate limit our requests when working with Notion from ratelimiter import RateLimiter # load the environment variables into global variables INSIGHTS_ID = os.environ["INSIGHTS_ID"] NOTION_KEY = os.environ["NOTION_KEY"] # classes for interacting with notion API class Client: def __init__(self, key): self.request = requests.Session() self.request.headers.update({ "Authorization": f"Bearer {key}", "Notion-Version": "2021-08-16", "Content-Type": "application/json" }) self.databases = Databases(self) self.pages = Pages(self) class Databases: def __init__(self, client): self.client = client def query(self, database_id): response = self.client.request.post(f"https://api.notion.com/v1/databases/{database_id}/query") return response.json() class Pages: def __init__(self, client): self.client = client def query_content(self, block_id, cursor=None): url = f"https://api.notion.com/v1/blocks/{block_id}/children?page_size=100" if cursor: url = f"{url}&start_cursor={cursor}" response = self.client.request.get(url) return response.json() client = Client(NOTION_KEY) # fetch database pages response = client.databases.query(INSIGHTS_ID) pages = response["results"] # fetch page content @RateLimiter(max_calls=3, period=1) def query_content_recursively(block_id, blocks, cursor=None): response = client.pages.query_content(block_id, cursor) for block in response["results"]: if block["has_children"]: blocks.append([block, query_content_recursively(block["id"], [])]) else: blocks.append(block) if response["next_cursor"]: return query_content_recursively(block_id, blocks, response["next_cursor"]) return blocks items = [] for page in pages: items.append({ "page": page, "blocks": query_content_recursively(page["id"], []) }) # extract data from page content def extract_title(page): return [t["plain_text"] for t in page["properties"]["Title"]["title"]] def extract_text(blocks, output): for block in blocks: if isinstance(block, list): extract_text(block, output) elif block["type"] in ["heading_1", "heading_2", "heading_3", "paragraph", "callout", "quote", "bulleted_list_item", "numbered_list_item", "to_do", "toggle"]: for text in block[block["type"]]["text"]: output.append(text["plain_text"]) return output def extract_relations(page): relations = set() for prop in page["properties"].values(): if prop["type"] == "relation": for relation in prop["relation"]: relations.add(relation["id"]) return relations def extract_mentions(blocks, mentions): for block in blocks: if isinstance(block, list): extract_mentions(block, mentions) elif block["type"] in ["heading_1", "heading_2", "heading_3", "paragraph", "callout", "quote", "bulleted_list_item", "numbered_list_item", "to_do", "toggle", "code"]: for text in block[block["type"]]["text"]: if text["type"] == "mention" and text["mention"]["type"] == "page": mentions.add(text["mention"]["page"]["id"]) elif block["type"] == "link_to_page": if block["link_to_page"]["type"] == "page_id": mentions.add(block["link_to_page"]["page_id"]) return mentions def extract_data(item): title = "".join(extract_title(item["page"])) text = " ".join(extract_text(item["blocks"], [])) relations = extract_relations(item["page"]) mentions = extract_mentions(item["blocks"], set()) return { "title": title, "text": text, "links": list(relations.union(mentions)) } for item in items: data = extract_data(item) item["title"] = data["title"] item["text"] = data["text"] item["links"] = data["links"]

originating_nodes = [] target_nodes = [] # create originating and target nodes for item in items: for link in item["links"]: # create bidirectional links originating_nodes.append(item["page"]["id"]) target_nodes.append(link) originating_nodes.append(link) target_nodes.append(item["page"]["id"]) # create dataframe using originating and target nodes df = pd.DataFrame({"originating": originating_nodes, "target": target_nodes}) # display preview df.head()

# create graph G = nx.from_pandas_edgelist(df, "originating", "target", create_using=nx.Graph()) # use page rank ppr1 = nx.pagerank(G) # print key-value pairs of node ids and their respective rank print(ppr1)

# sort pages sorted_pages = sorted(ppr1.items(), key=lambda x: x[1], reverse=True)[:10] # print out sorted pages for page_id, score in sorted_pages: result = [item["title"] for item in items if item["page"]["id"] == page_id] if (len(result) > 0): print(score, result)