inoreader2readwise/job/main.py

import os
import time
import json
import requests
import logging

DATA_STORE_PATH = "/data/last_update_time.txt"

DATABASE_URL = os.getenv("DATABASE_URL")

logging.basicConfig(level=logging.INFO)

class APIHandler:
    def __init__(self, base_url, headers={}):
        self.base_url = base_url
        self.headers = headers

    def get(self, endpoint, params=None):
        response = requests.get(self.base_url + endpoint, params=params, headers=self.headers)
        response.raise_for_status()
        return response.json()

    def post(self, endpoint, data=None):
        response = requests.post(self.base_url + endpoint, data=json.dumps(data), headers=self.headers)
        response.raise_for_status()
        return response.status_code

def get_last_update_time():
    with open(DATA_STORE_PATH, 'r') as file:
        return int(file.read().strip())

def update_last_update_time(new_time):
    with open(DATA_STORE_PATH, 'w') as file:
        file.write(str(new_time))

def get_new_annotations(last_annotation_time):
    inoreader = APIHandler(
        "https://www.inoreader.com/reader/api/0/stream/contents",
        headers = {
            'Authorization': 'Bearer ' + get_inoreader_access_token()
        }
    )

    all_annotations = []
    continuation = None

    while True:
        params = {
            "annotations": 1,
            "n": 100,
        }
        if continuation:
            params["c"] = continuation

        inoreader_response = inoreader.get(
            "/user/-/state/com.google/annotated",
            params=params
        )
        data = json.loads(inoreader_response)

        for item in data["items"]:
            annotations = item.get("annotations", [])
            for annotation in annotations:
                annotation['title'] = item['title']
                annotation['author'] = item['author']
                annotation['sources'] = item['canonical']
                all_annotations.append(annotation)

        if 'continuation' in data:
            continuation = data['continuation']
            time.sleep(900)  # Sleep for 15 minutes between pages
        else:
            break

    return [annotation for annotation in all_annotations if annotation['added_on'] > last_annotation_time]

def push_annotations_to_readwise(annotations):
    readwise = APIHandler(
        "https://readwise.io",
        headers = {
            'Authorization': 'Token ' + os.getenv("READWISE_ACCESS_TOKEN"),
            'Content-Type': 'application/json'
        }
    )

    readwise.post(
        "/api/v2/highlights/",
        data={
            'highlights': [
                {
                    'text': annotation['text'],
                    'title': annotation['title'],
                    'author': annotation['author'],
                    'note': annotation['note'],
                    'highlighted_at': annotation['added_on'],
                    'category': 'articles',
                    'source_url': annotation['sources'][0]['href'] if annotation['sources'] else None,
                }
                for annotation in annotations
            ]
        }
    )

def get_inoreader_access_token():
    response = requests.get(f'{DATABASE_URL}/token/latest')
    response.raise_for_status()

    if response.status_code == 204:
        # throw error - not logged in. Please log in first through the web app
        raise Exception("Not logged in. Please log in first through the web app")
    elif response.status_code == 200:
        resp_json = response.json()
        if resp_json['token']['expiration_seconds'] + resp_json['token']['timestamp'] > datetime.now().timestamp():
            return resp_json['token']['access_token']
        else:
            return refresh_inoreader_access_token(resp_json['token']['refresh_token'])

    access_token = get_token_from_database()
    if not access_token:
        access_token = refresh_inoreader_access_token()

    if not access_token:
        raise Exception("Unable to get access token. Try logging in again through the web app")
    return access_token

def refresh_inoreader_access_token(refresh_token):
    response = requests.post(
        'https://www.inoreader.com/oauth2/token',
        headers={
            'Content-Type': 'application/x-www-form-urlencoded',
        },
        data={
            'refresh_token': refresh_token,
            'client_id': os.getenv("INOREADER_CLIENT_ID"),
            'client_secret': os.getenv("INOREADER_CLIENT_SECRET"),
            'grant_type': 'refresh_token'
        }
    )

    response.raise_for_status()

    tokens = response.json()

    # Save tokens for later use
    save_tokens(tokens['access_token'], tokens['refresh_token'], tokens['expires_in'])

    return tokens['access_token']

def save_tokens(access_token, refresh_token, expiration_seconds):
    response = requests.post(
        f'{DATABASE_URL}/token',
        headers={
            'Content-Type': 'application/json'
        },
        json={
            'access_token': access_token,
            'refresh_token': refresh_token,
            'expiration_seconds': expiration_seconds
        }
    )
    response.raise_for_status()

def main():

    while True:
        try:
            last_annotation_time = get_last_update_time()
            new_annotations = get_new_annotations(last_annotation_time)

            if new_annotations:
                latest_added_on = max(annotation['added_on'] for annotation in new_annotations)
                push_annotations_to_readwise(new_annotations)
                update_last_update_time(latest_added_on)
            else:
                logging.info("No new annotations found")

            time.sleep(86400)  # Sleep for 24 hours

        except Exception as e:
            logging.error(f"An error occurred: {e}")
            time.sleep(3600)  # Sleep for 1 hour in case of error

if __name__ == "__main__":
    main()