Added jobs
This commit is contained in:
parent
6609f8a054
commit
56061d5361
GPG Key ID:
81C942771BB69898
242
job/main.py
Normal file
242
job/main.py
Normal file
|
|
@ -0,0 +1,242 @@
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
import json
|
||||||
|
import requests
|
||||||
|
import logging
|
||||||
|
|
||||||
|
DATA_STORE_PATH = "/data/last_update_time.txt"
|
||||||
|
|
||||||
|
DATABASE_URL = os.getenv("DATABASE_URL")
|
||||||
|
|
||||||
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
|
||||||
|
class APIHandler:
|
||||||
|
def __init__(self, base_url, headers={}):
|
||||||
|
self.base_url = base_url
|
||||||
|
self.headers = headers
|
||||||
|
|
||||||
|
def get(self, endpoint, params=None):
|
||||||
|
response = requests.get(self.base_url + endpoint, params=params, headers=self.headers)
|
||||||
|
response.raise_for_status()
|
||||||
|
return response.json()
|
||||||
|
|
||||||
|
def post(self, endpoint, data=None):
|
||||||
|
response = requests.post(self.base_url + endpoint, data=json.dumps(data), headers=self.headers)
|
||||||
|
response.raise_for_status()
|
||||||
|
return response.status_code
|
||||||
|
|
||||||
|
def get_last_update_time(email):
|
||||||
|
response = requests.get(f'{DATABASE_URL}/annotation_last_update/{email}')
|
||||||
|
response.raise_for_status()
|
||||||
|
|
||||||
|
if response.status_code == 204:
|
||||||
|
return 0
|
||||||
|
elif response.status_code == 200:
|
||||||
|
return response.json()['last_update_time']
|
||||||
|
|
||||||
|
def update_last_update_time(email, new_time):
|
||||||
|
response = requests.post(
|
||||||
|
f'{DATABASE_URL}/annotation_last_update',
|
||||||
|
headers={
|
||||||
|
'Content-Type': 'application/json'
|
||||||
|
},
|
||||||
|
json={
|
||||||
|
'email': email,
|
||||||
|
'last_update_time': new_time
|
||||||
|
}
|
||||||
|
)
|
||||||
|
response.raise_for_status()
|
||||||
|
|
||||||
|
def get_new_annotations(last_annotation_time, inoreader_token):
|
||||||
|
inoreader = APIHandler(
|
||||||
|
"https://www.inoreader.com/reader/api/0/stream/contents",
|
||||||
|
headers = {
|
||||||
|
'Authorization': 'Bearer ' + inoreader_token()
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
all_annotations = []
|
||||||
|
continuation = None
|
||||||
|
|
||||||
|
while True:
|
||||||
|
params = {
|
||||||
|
"annotations": 1,
|
||||||
|
"n": 100,
|
||||||
|
}
|
||||||
|
if continuation:
|
||||||
|
params["c"] = continuation
|
||||||
|
|
||||||
|
inoreader_response = inoreader.get(
|
||||||
|
"/user/-/state/com.google/annotated",
|
||||||
|
params=params
|
||||||
|
)
|
||||||
|
data = json.loads(inoreader_response)
|
||||||
|
|
||||||
|
for item in data["items"]:
|
||||||
|
annotations = item.get("annotations", [])
|
||||||
|
for annotation in annotations:
|
||||||
|
annotation['title'] = item['title']
|
||||||
|
annotation['author'] = item['author']
|
||||||
|
annotation['sources'] = item['canonical']
|
||||||
|
all_annotations.append(annotation)
|
||||||
|
|
||||||
|
if 'continuation' in data:
|
||||||
|
continuation = data['continuation']
|
||||||
|
time.sleep(900) # Sleep for 15 minutes between pages
|
||||||
|
else:
|
||||||
|
break
|
||||||
|
|
||||||
|
return [annotation for annotation in all_annotations if annotation['added_on'] > last_annotation_time]
|
||||||
|
|
||||||
|
def push_annotations_to_readwise(annotations, readwise_token):
|
||||||
|
readwise = APIHandler(
|
||||||
|
"https://readwise.io",
|
||||||
|
headers = {
|
||||||
|
'Authorization': 'Token ' + readwise_token,
|
||||||
|
'Content-Type': 'application/json'
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
readwise.post(
|
||||||
|
"/api/v2/highlights/",
|
||||||
|
data={
|
||||||
|
'highlights': [
|
||||||
|
{
|
||||||
|
'text': annotation['text'],
|
||||||
|
'title': annotation['title'],
|
||||||
|
'author': annotation['author'],
|
||||||
|
'note': annotation['note'],
|
||||||
|
'highlighted_at': annotation['added_on'],
|
||||||
|
'category': 'articles',
|
||||||
|
'source_url': annotation['sources'][0]['href'] if annotation['sources'] else None,
|
||||||
|
}
|
||||||
|
for annotation in annotations
|
||||||
|
]
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
# def get_inoreader_access_token():
|
||||||
|
# response = requests.get(f'{DATABASE_URL}/token/latest')
|
||||||
|
# response.raise_for_status()
|
||||||
|
|
||||||
|
# if response.status_code == 204:
|
||||||
|
# # throw error - not logged in. Please log in first through the web app
|
||||||
|
# raise Exception("Not logged in. Please log in first through the web app")
|
||||||
|
# elif response.status_code == 200:
|
||||||
|
# resp_json = response.json()
|
||||||
|
# if resp_json['token']['expiration_seconds'] + resp_json['token']['timestamp'] > datetime.now().timestamp():
|
||||||
|
# return resp_json['token']['access_token']
|
||||||
|
# else:
|
||||||
|
# return refresh_inoreader_access_token(resp_json['token']['refresh_token'])
|
||||||
|
|
||||||
|
# access_token = get_token_from_database()
|
||||||
|
# if not access_token:
|
||||||
|
# access_token = refresh_inoreader_access_token()
|
||||||
|
|
||||||
|
# if not access_token:
|
||||||
|
# raise Exception("Unable to get access token. Try logging in again through the web app")
|
||||||
|
# return access_token
|
||||||
|
|
||||||
|
def refresh_inoreader_access_token(refresh_token, readwise_api_key):
|
||||||
|
response = requests.post(
|
||||||
|
'https://www.inoreader.com/oauth2/token',
|
||||||
|
headers={
|
||||||
|
'Content-Type': 'application/x-www-form-urlencoded',
|
||||||
|
},
|
||||||
|
data={
|
||||||
|
'refresh_token': refresh_token,
|
||||||
|
'client_id': os.getenv("INOREADER_CLIENT_ID"),
|
||||||
|
'client_secret': os.getenv("INOREADER_CLIENT_SECRET"),
|
||||||
|
'grant_type': 'refresh_token'
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
response.raise_for_status()
|
||||||
|
|
||||||
|
token = response.json()
|
||||||
|
|
||||||
|
user_info = requests.get('https://www.inoreader.com/reader/api/0/user-info', headers={
|
||||||
|
'Authorization': f'Bearer {token.get("access_token")}'
|
||||||
|
}).json()
|
||||||
|
|
||||||
|
# Save tokens for later use
|
||||||
|
save_token(
|
||||||
|
user_info.get('userEmail'),
|
||||||
|
token['access_token'],
|
||||||
|
token['refresh_token'],
|
||||||
|
token['expires_in'],
|
||||||
|
readwise_api_key
|
||||||
|
)
|
||||||
|
|
||||||
|
return token['access_token'], readwise_api_key
|
||||||
|
|
||||||
|
def save_token(email, access_token, refresh_token, expiration_seconds, readwise_api_key):
|
||||||
|
response = requests.post(
|
||||||
|
f'{DATABASE_URL}/token',
|
||||||
|
headers={
|
||||||
|
'Content-Type': 'application/json'
|
||||||
|
},
|
||||||
|
json={
|
||||||
|
'email': email,
|
||||||
|
'access_token': access_token,
|
||||||
|
'refresh_token': refresh_token,
|
||||||
|
'expiration_seconds': expiration_seconds,
|
||||||
|
'readwise_api_key': readwise_api_key
|
||||||
|
}
|
||||||
|
)
|
||||||
|
response.raise_for_status()
|
||||||
|
|
||||||
|
def get_all_active_tokens():
|
||||||
|
response = requests.get(f'{DATABASE_URL}/token/all?only_active=true')
|
||||||
|
response.raise_for_status()
|
||||||
|
|
||||||
|
if response.status_code == 200:
|
||||||
|
return response.json()['tokens']
|
||||||
|
else:
|
||||||
|
return []
|
||||||
|
|
||||||
|
def deactivate_token(token_id):
|
||||||
|
response = requests.post(
|
||||||
|
f'{DATABASE_URL}/token/{token_id}/deactivate',
|
||||||
|
headers={
|
||||||
|
'Content-Type': 'application/json'
|
||||||
|
}
|
||||||
|
)
|
||||||
|
response.raise_for_status()
|
||||||
|
|
||||||
|
def check_and_refresh_access_token(token):
|
||||||
|
if token['expiration_seconds'] + token['timestamp'] > datetime.now().timestamp():
|
||||||
|
return token['access_token'], token['readwise_api_key']
|
||||||
|
else:
|
||||||
|
deactivate_token(token['id'])
|
||||||
|
return refresh_inoreader_access_token(token['refresh_token'], token['readwise_api_key'])
|
||||||
|
|
||||||
|
def main():
|
||||||
|
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
all_tokens = get_all_active_tokens()
|
||||||
|
for token in all_tokens:
|
||||||
|
logging.info("Checking for new annotations for user with email: {}".format(token['email']))
|
||||||
|
|
||||||
|
inoreader_token, readwise_api_key = check_and_refresh_access_token(token)
|
||||||
|
|
||||||
|
last_annotation_time = get_last_update_time(token['email'])
|
||||||
|
new_annotations = get_new_annotations(last_annotation_time, inoreader_token)
|
||||||
|
|
||||||
|
if new_annotations:
|
||||||
|
latest_added_on = max(annotation['added_on'] for annotation in new_annotations)
|
||||||
|
push_annotations_to_readwise(new_annotations, readwise_api_key)
|
||||||
|
update_last_update_time(token['email'], latest_added_on)
|
||||||
|
logging.info("Successfully pushed {} new annotations to Readwise for user with email: {}".format(len(new_annotations), token['email']))
|
||||||
|
else:
|
||||||
|
logging.info("No new annotations found for user with email: {}".format(token['email']))
|
||||||
|
|
||||||
|
time.sleep(86400) # Sleep for 24 hours
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logging.error(f"An error occurred: {e}")
|
||||||
|
time.sleep(3600) # Sleep for 1 hour in case of error
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
1
job/requirements.txt
Normal file
1
job/requirements.txt
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
requests==2.31.0
|
||||||
Loading…
Reference in New Issue
Block a user