inoreader2readwise/job/main.py
2024-01-24 16:59:35 +05:30

184 lines
5.8 KiB
Python

import os
import time
import json
import requests
import logging
DATA_STORE_PATH = "/data/last_update_time.txt"
DATABASE_URL = os.getenv("DATABASE_URL")
logging.basicConfig(level=logging.INFO)
class APIHandler:
def __init__(self, base_url, headers={}):
self.base_url = base_url
self.headers = headers
def get(self, endpoint, params=None):
response = requests.get(self.base_url + endpoint, params=params, headers=self.headers)
response.raise_for_status()
return response.json()
def post(self, endpoint, data=None):
response = requests.post(self.base_url + endpoint, data=json.dumps(data), headers=self.headers)
response.raise_for_status()
return response.status_code
def get_last_update_time():
with open(DATA_STORE_PATH, 'r') as file:
return int(file.read().strip())
def update_last_update_time(new_time):
with open(DATA_STORE_PATH, 'w') as file:
file.write(str(new_time))
def get_new_annotations(last_annotation_time):
inoreader = APIHandler(
"https://www.inoreader.com/reader/api/0/stream/contents",
headers = {
'Authorization': 'Bearer ' + get_inoreader_access_token()
}
)
all_annotations = []
continuation = None
while True:
params = {
"annotations": 1,
"n": 100,
}
if continuation:
params["c"] = continuation
inoreader_response = inoreader.get(
"/user/-/state/com.google/annotated",
params=params
)
data = json.loads(inoreader_response)
for item in data["items"]:
annotations = item.get("annotations", [])
for annotation in annotations:
annotation['title'] = item['title']
annotation['author'] = item['author']
annotation['sources'] = item['canonical']
all_annotations.append(annotation)
if 'continuation' in data:
continuation = data['continuation']
time.sleep(900) # Sleep for 15 minutes between pages
else:
break
return [annotation for annotation in all_annotations if annotation['added_on'] > last_annotation_time]
def push_annotations_to_readwise(annotations):
readwise = APIHandler(
"https://readwise.io",
headers = {
'Authorization': 'Token ' + os.getenv("READWISE_ACCESS_TOKEN"),
'Content-Type': 'application/json'
}
)
readwise.post(
"/api/v2/highlights/",
data={
'highlights': [
{
'text': annotation['text'],
'title': annotation['title'],
'author': annotation['author'],
'note': annotation['note'],
'highlighted_at': annotation['added_on'],
'category': 'articles',
'source_url': annotation['sources'][0]['href'] if annotation['sources'] else None,
}
for annotation in annotations
]
}
)
def get_inoreader_access_token():
response = requests.get(f'{DATABASE_URL}/token/latest')
response.raise_for_status()
if response.status_code == 204:
# throw error - not logged in. Please log in first through the web app
raise Exception("Not logged in. Please log in first through the web app")
elif response.status_code == 200:
resp_json = response.json()
if resp_json['token']['expiration_seconds'] + resp_json['token']['timestamp'] > datetime.now().timestamp():
return resp_json['token']['access_token']
else:
return refresh_inoreader_access_token(resp_json['token']['refresh_token'])
access_token = get_token_from_database()
if not access_token:
access_token = refresh_inoreader_access_token()
if not access_token:
raise Exception("Unable to get access token. Try logging in again through the web app")
return access_token
def refresh_inoreader_access_token(refresh_token):
response = requests.post(
'https://www.inoreader.com/oauth2/token',
headers={
'Content-Type': 'application/x-www-form-urlencoded',
},
data={
'refresh_token': refresh_token,
'client_id': os.getenv("INOREADER_CLIENT_ID"),
'client_secret': os.getenv("INOREADER_CLIENT_SECRET"),
'grant_type': 'refresh_token'
}
)
response.raise_for_status()
tokens = response.json()
# Save tokens for later use
save_tokens(tokens['access_token'], tokens['refresh_token'], tokens['expires_in'])
return tokens['access_token']
def save_tokens(access_token, refresh_token, expiration_seconds):
response = requests.post(
f'{DATABASE_URL}/token',
headers={
'Content-Type': 'application/json'
},
json={
'access_token': access_token,
'refresh_token': refresh_token,
'expiration_seconds': expiration_seconds
}
)
response.raise_for_status()
def main():
while True:
try:
last_annotation_time = get_last_update_time()
new_annotations = get_new_annotations(last_annotation_time)
if new_annotations:
latest_added_on = max(annotation['added_on'] for annotation in new_annotations)
push_annotations_to_readwise(new_annotations)
update_last_update_time(latest_added_on)
else:
logging.info("No new annotations found")
time.sleep(86400) # Sleep for 24 hours
except Exception as e:
logging.error(f"An error occurred: {e}")
time.sleep(3600) # Sleep for 1 hour in case of error
if __name__ == "__main__":
main()