import os import time import json import requests import logging DATA_STORE_PATH = "/data/last_update_time.txt" DATABASE_URL = os.getenv("DATABASE_URL") logging.basicConfig(level=logging.INFO) class APIHandler: def __init__(self, base_url, headers={}): self.base_url = base_url self.headers = headers def get(self, endpoint, params=None): response = requests.get(self.base_url + endpoint, params=params, headers=self.headers) response.raise_for_status() return response.json() def post(self, endpoint, data=None): response = requests.post(self.base_url + endpoint, data=json.dumps(data), headers=self.headers) response.raise_for_status() return response.status_code def get_last_update_time(email): response = requests.get(f'{DATABASE_URL}/annotation_last_update/{email}') response.raise_for_status() if response.status_code == 204: return 0 elif response.status_code == 200: return response.json()['last_update_time'] def update_last_update_time(email, new_time): response = requests.post( f'{DATABASE_URL}/annotation_last_update', headers={ 'Content-Type': 'application/json' }, json={ 'email': email, 'last_update_time': new_time } ) response.raise_for_status() def get_new_annotations(last_annotation_time, inoreader_token): inoreader = APIHandler( "https://www.inoreader.com/reader/api/0/stream/contents", headers = { 'Authorization': 'Bearer ' + inoreader_token() } ) all_annotations = [] continuation = None while True: params = { "annotations": 1, "n": 100, } if continuation: params["c"] = continuation inoreader_response = inoreader.get( "/user/-/state/com.google/annotated", params=params ) data = json.loads(inoreader_response) for item in data["items"]: annotations = item.get("annotations", []) for annotation in annotations: annotation['title'] = item['title'] annotation['author'] = item['author'] annotation['sources'] = item['canonical'] all_annotations.append(annotation) if 'continuation' in data: continuation = data['continuation'] time.sleep(900) # Sleep for 15 minutes between pages else: break return [annotation for annotation in all_annotations if annotation['added_on'] > last_annotation_time] def push_annotations_to_readwise(annotations, readwise_token): readwise = APIHandler( "https://readwise.io", headers = { 'Authorization': 'Token ' + readwise_token, 'Content-Type': 'application/json' } ) readwise.post( "/api/v2/highlights/", data={ 'highlights': [ { 'text': annotation['text'], 'title': annotation['title'], 'author': annotation['author'], 'note': annotation['note'], 'highlighted_at': annotation['added_on'], 'category': 'articles', 'source_url': annotation['sources'][0]['href'] if annotation['sources'] else None, } for annotation in annotations ] } ) # def get_inoreader_access_token(): # response = requests.get(f'{DATABASE_URL}/token/latest') # response.raise_for_status() # if response.status_code == 204: # # throw error - not logged in. Please log in first through the web app # raise Exception("Not logged in. Please log in first through the web app") # elif response.status_code == 200: # resp_json = response.json() # if resp_json['token']['expiration_seconds'] + resp_json['token']['timestamp'] > datetime.now().timestamp(): # return resp_json['token']['access_token'] # else: # return refresh_inoreader_access_token(resp_json['token']['refresh_token']) # access_token = get_token_from_database() # if not access_token: # access_token = refresh_inoreader_access_token() # if not access_token: # raise Exception("Unable to get access token. Try logging in again through the web app") # return access_token def refresh_inoreader_access_token(refresh_token, readwise_api_key): response = requests.post( 'https://www.inoreader.com/oauth2/token', headers={ 'Content-Type': 'application/x-www-form-urlencoded', }, data={ 'refresh_token': refresh_token, 'client_id': os.getenv("INOREADER_CLIENT_ID"), 'client_secret': os.getenv("INOREADER_CLIENT_SECRET"), 'grant_type': 'refresh_token' } ) response.raise_for_status() token = response.json() user_info = requests.get('https://www.inoreader.com/reader/api/0/user-info', headers={ 'Authorization': f'Bearer {token.get("access_token")}' }).json() # Save tokens for later use save_token( user_info.get('userEmail'), token['access_token'], token['refresh_token'], token['expires_in'], readwise_api_key ) return token['access_token'], readwise_api_key def save_token(email, access_token, refresh_token, expiration_seconds, readwise_api_key): response = requests.post( f'{DATABASE_URL}/token', headers={ 'Content-Type': 'application/json' }, json={ 'email': email, 'access_token': access_token, 'refresh_token': refresh_token, 'expiration_seconds': expiration_seconds, 'readwise_api_key': readwise_api_key } ) response.raise_for_status() def get_all_active_tokens(): response = requests.get(f'{DATABASE_URL}/token/all?only_active=true') response.raise_for_status() if response.status_code == 200: return response.json()['tokens'] else: return [] def deactivate_token(token_id): response = requests.post( f'{DATABASE_URL}/token/{token_id}/deactivate', headers={ 'Content-Type': 'application/json' } ) response.raise_for_status() def check_and_refresh_access_token(token): if token['expiration_seconds'] + token['timestamp'] > datetime.now().timestamp(): return token['access_token'], token['readwise_api_key'] else: deactivate_token(token['id']) return refresh_inoreader_access_token(token['refresh_token'], token['readwise_api_key']) def main(): while True: try: all_tokens = get_all_active_tokens() for token in all_tokens: logging.info("Checking for new annotations for user with email: {}".format(token['email'])) inoreader_token, readwise_api_key = check_and_refresh_access_token(token) last_annotation_time = get_last_update_time(token['email']) new_annotations = get_new_annotations(last_annotation_time, inoreader_token) if new_annotations: latest_added_on = max(annotation['added_on'] for annotation in new_annotations) push_annotations_to_readwise(new_annotations, readwise_api_key) update_last_update_time(token['email'], latest_added_on) logging.info("Successfully pushed {} new annotations to Readwise for user with email: {}".format(len(new_annotations), token['email'])) else: logging.info("No new annotations found for user with email: {}".format(token['email'])) time.sleep(86400) # Sleep for 24 hours except Exception as e: logging.error(f"An error occurred: {e}") time.sleep(3600) # Sleep for 1 hour in case of error if __name__ == "__main__": main()