Compare commits

...

44 Commits

Author SHA1 Message Date
Shailaja Kumari 916828a05c udated code for testing 2024-03-01 15:23:15 +05:30
Shailaja Kumari f3a096e8c1
. 2024-02-15 20:35:54 +05:30
Shailaja Kumari 6da23cb97d
. 2024-02-15 11:54:09 +05:30
Shailaja Kumari c2ad41f4fd
. 2024-02-15 08:33:44 +05:30
Shailaja Kumari 1f5d216faa
correction 2024-02-15 08:26:15 +05:30
Shailaja Kumari fe531ef859
service/part3 (url encoding) 2024-02-15 08:15:03 +05:30
Shailaja Kumari b5373d144e
url encoding in python 2024-02-14 10:34:21 +05:30
Shailaja Kumari bde73bf535
added print statement 2024-02-13 22:30:54 +05:30
Swapnil a43479276b
fixes: constant AUTH_URL & better csrf handeling 2024-02-07 12:43:26 +05:30
Swapnil 10dab44a27
app: update dummy.env 2024-02-01 16:37:36 +05:30
Swapnil 68869249bc
app: fix user_info error 2024-02-01 16:37:36 +05:30
Swapnil dd850ca6e1
app: minor improvements 2024-02-01 16:37:35 +05:30
Swapnil 706ff9d90f
db: Table and apis for last annotation time 2024-02-01 16:37:35 +05:30
Swapnil 889469dcff
job: get/update last annotation time from database 2024-02-01 16:37:35 +05:30
Swapnil 1c0f991977
added dummy.env for reference 2024-02-01 16:37:34 +05:30
Swapnil ee9b4b10a1
job: Run job for all active tokens
- check, deactivate, & refresh token
- better logging
2024-02-01 16:37:34 +05:30
Swapnil 65656ff559
db: API to deactivate token & fixes 2024-02-01 16:37:34 +05:30
Swapnil e80e88bb99
app: Comments - inoreader user-info 2024-02-01 16:37:33 +05:30
Swapnil 46e4f8e722
app: fix None readwise_api_key 2024-02-01 16:37:33 +05:30
Swapnil 19af4bff34
db: api to get all entires 2024-02-01 16:37:33 +05:30
Swapnil c1b216dd11
app: fix logic to save/create token 2024-02-01 16:37:33 +05:30
Swapnil 0c1e18cad9
db: new columns & APIs 2024-02-01 16:37:32 +05:30
Swapnil 97c3354de8
app: minor fixes 2024-02-01 16:37:32 +05:30
Swapnil 84a1ed15f5
app: fix 2024-02-01 16:37:32 +05:30
Swapnil 8fd838ce23
app: fix error 2024-02-01 16:37:31 +05:30
Swapnil 20423379a2
app: fix error 2024-02-01 16:37:31 +05:30
Swapnil f34b85d79f
fixes 2024-02-01 16:37:31 +05:30
Swapnil 91ac98a60b
fixes 2024-02-01 16:37:30 +05:30
Swapnil 327896abcc
app: fixed bug and better error message 2024-02-01 16:37:30 +05:30
Swapnil bf8b33edde
app: try better error in app 2024-02-01 16:37:30 +05:30
Swapnil cce57bca72
db-service: fix required fields checking 2024-02-01 16:37:29 +05:30
Swapnil 3ec293e1fa
app-service: Updated templates and login logic 2024-02-01 16:37:29 +05:30
Swapnil 1b7a37c4e1
db-service: Updated table schema & exposed new APIs 2024-02-01 16:37:29 +05:30
Swapnil 67766a7708
fix logout method 2024-02-01 16:37:28 +05:30
Swapnil 021f3bcb7c
use app.secret_key to enable session 2024-02-01 16:37:28 +05:30
Swapnil 6614f74c9f
fix database api 2024-02-01 16:37:28 +05:30
Swapnil d0c888d6c4
implement logout 2024-02-01 16:37:27 +05:30
Swapnil 16c94fcaa6
test with github auth 2024-02-01 16:37:27 +05:30
Swapnil 516d5db9a0
fix: app - import datetime 2024-02-01 16:37:27 +05:30
Swapnil 29a39cb633
fix: database - timestamp 2024-02-01 16:37:27 +05:30
Swapnil 1d1c2aac9f
fix: app - expose on 0.0.0.0 2024-02-01 16:37:26 +05:30
Swapnil c377909954
fix: database check 2024-02-01 16:37:26 +05:30
Swapnil 34282e6f30
fix: with app.app_context() 2024-02-01 16:37:26 +05:30
Swapnil 089583960b
Separate micro services 2024-02-01 16:37:25 +05:30
14 changed files with 718 additions and 125 deletions

11
app/Dockerfile Normal file
View File

@ -0,0 +1,11 @@
FROM python:3.8-slim-buster
WORKDIR /app
ADD . /app
RUN pip install --no-cache-dir -r requirements.txt
EXPOSE 5000
CMD ["python", "main.py"]

8
app/dummy.env Normal file
View File

@ -0,0 +1,8 @@
CLIENT_ID=
CLIENT_SECRET=
REDIRECT_URI=
OPTIONAL_SCOPES=
DATABASE_URL=
# generated by `openssl rand -hex 24` - used to encrypt session
APP_SECRET_KEY=

198
app/main.py Normal file
View File

@ -0,0 +1,198 @@
import os
from flask import Flask, render_template, request, redirect, abort, url_for, session
import requests
from datetime import datetime
from urllib.parse import urlencode
def get_env_variable(var_name):
value = os.environ.get(var_name)
if not value:
raise ValueError(f"Missing required environment variable: {var_name}")
return value
app = Flask(__name__)
# Read environment variables outside the route function
client_id = get_env_variable('CLIENT_ID')
client_secret = get_env_variable('CLIENT_SECRET')
redirect_uri = get_env_variable('REDIRECT_URI')
optional_scopes = get_env_variable('OPTIONAL_SCOPES')
database_url = get_env_variable('DATABASE_URL')
secret_key = get_env_variable('APP_SECRET_KEY')
# Set secret key to enable sessions
app.secret_key = secret_key
# https://www.inoreader.com/oauth2/auth
# Corrected URL for Inoreader OAuth
# AUTH_URL = 'https://www.inoreader.com/oauth2/auth'
AUTH_URL = 'https://github.com/login/oauth/authorize'
#defining constant
# TOKEN_URL = 'https://www.inoreader.com/oauth2/token'
# USER_INFO_URL = 'https://www.inoreader.com/reader/api/0/user-info'
@app.route('/')
def home():
if is_logged_in():
return main_menu()
else:
return generate_login_page()
def main_menu():
token_id = session.get('token_id')
token = get_token_from_database(token_id)
user_info = get_user_info(token['access_token'])
last_synced, next_sync = format_sync_times(token)
return render_template('home.html', user_login=user_info.get('userName'),
user_email=user_info.get('userEmail'),
readwise_api_key=token.get('readwise_api_key', ''),
last_synced=last_synced, next_sync=next_sync)
def generate_login_page():
session['csrf_protection_string'] = os.urandom(16).hex()
oauth_params = {
'client_id': client_id,
'redirect_uri': redirect_uri,
'response_type': 'code',
'scope': optional_scopes,
'state': session['csrf_protection_string']
}
oauth_url = f'{AUTH_URL}?{urlencode(oauth_params)}'
return render_template('login.html', oauth_url=oauth_url)
@app.route('/oauth-redirect')
def oauth_redirect():
auth_code = request.args.get('code')
csrf_token = request.args.get('state')
# Verify the CSRF protection string
if csrf_token != session.get('csrf_protection_string'):
abort(403, 'Invalid CSRF token. Please try again.')
# Exchange authorization code for access and refresh tokens using the Inoreader API
response = requests.post(
https://github.com/login/oauth/access_token',
headers={'Accept': 'application/json'},
data={
'code': auth_code,
'redirect_uri': redirect_uri,
'client_id': client_id,
'client_secret': client_secret,
}
)
raise_for_status(response)
token = response.json()
# Fetch user information from Inoreader
user_info = requests.get(https://api.github.com/user', headers={
'Authorization': f'Bearer {token.get("access_token")}'
}).json()
# Save tokens for later use
token_id = save_or_update_token(user_info.get('userEmail'), token)
set_session_token_id(token_id)
return redirect(url_for('home'))
# logout
@app.route('/logout', methods=['POST'])
def logout():
token_id = session.get('token_id')
if not token_id:
return redirect(url_for('home'))
# remove token_id from session
session.pop('token_id', None)
# response = requests.put(f'{database_url}/token/{token_id}', headers={
# 'Content-Type': 'application/json'
# }, json={
# 'is_logged_in': False
# })
# response.raise_for_status()
return redirect(url_for('home'))
@app.route('/readwise', methods=['POST'])
def submit_readwise_api():
token_id = session.get('token_id')
if not token_id:
return redirect(url_for('home'))
response = requests.put(f'{database_url}/token/{token_id}', headers={
'Content-Type': 'application/json'
}, json={
'readwise_api_key': request.form.get('readwise_api_key')
})
raise_for_status(response)
return redirect(url_for('home'))
def is_logged_in():
token_id = session.get('token_id')
if not token_id:
return False
response = requests.get(f'{database_url}/token/{token_id}')
raise_for_status(response)
resp_json = response.json()
token = resp_json['token']
return token.get('active', False)
def save_or_update_token(email, access_token, refresh_token, expiration_seconds):
response = requests.get(f'{database_url}/token?email={email}')
raise_for_status(response)
if response.status_code == 200:
update_login(response.json()['token']['id'], access_token, refresh_token, expiration_seconds)
else:
add_login(email, access_token, refresh_token, expiration_seconds)
def add_login(email, access_token, refresh_token, expiration_seconds):
response = requests.post(
f'{database_url}/token',
headers={'Content-Type': 'application/json'},
json={
'email': email,
'access_token': access_token,
'refresh_token': refresh_token,
'expiration_seconds': expiration_seconds
}
)
raise_for_status(response)
return response.json().get('id')
def update_login(token_id, access_token, refresh_token, expiration_seconds):
response = requests.put(
f'{database_url}/token/{token_id}',
headers={'Content-Type': 'application/json'},
json={
'access_token': access_token,
'refresh_token': refresh_token,
'expiration_seconds': expiration_seconds
}
)
raise_for_status(response)
return token_id
def set_session_token_id(token_id):
session['token_id'] = token_id
def raise_for_status(response):
if response.status_code not in range(200, 300):
try:
msg = response.json().get('error', 'No error message provided')
except Exception:
msg = response.text
raise Exception(f'HTTPError: {response.status_code} - Message: {msg}')
if __name__ == '__main__':
app.run(host='0.0.0.0', debug=True, port=5000)

2
app/requirements.txt Normal file
View File

@ -0,0 +1,2 @@
Flask==3.0.1
requests==2.31.0

28
app/templates/home.html Normal file
View File

@ -0,0 +1,28 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Inoreader To Readwise</title>
</head>
<body>
<h1>Logged In as {{ user_login }} - {{ user_email }}</h1>
<!-- show last synced and next synced time -->
<p>Last Synced: {{ last_synced }}</p>
<p>Next Sync: {{ next_sync }}</p>
<br>
<!-- Take readiwse api key input -->
<form action="/readwise" method="POST">
<label for="api_key">Readwise API Key</label>
<input type="text" name="readwise_api_key" id="api_key", value="{{ readwise_api_key }}" required>
<input type="submit" value="Submit">
</form>
<!-- Logout -->
<form action="/logout" method="POST">
<input type="submit" value="Logout">
</form>
</body>
</html>

13
app/templates/login.html Normal file
View File

@ -0,0 +1,13 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Inoreader To Readwise</title>
</head>
<body>
<!-- <button onclick="redirectToOAuth()">Login using inoreader</button> -->
<a href="{{oauth_url}}">Login via InoReader</a>
</body>
</html>

11
database/Dockerfile Normal file
View File

@ -0,0 +1,11 @@
FROM python:3.8-slim-buster
WORKDIR /app
ADD . /app
RUN pip install --no-cache-dir -r requirements.txt
EXPOSE 5000
CMD ["python", "main.py"]

200
database/main.py Normal file
View File

@ -0,0 +1,200 @@
from flask import Flask, jsonify, request
from flask_sqlalchemy import SQLAlchemy
from datetime import datetime
import uuid
app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///tokens.db' # Use SQLite for simplicity
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
db = SQLAlchemy(app)
class Token(db.Model):
id = db.Column(db.String(36), primary_key=True, default=str(uuid.uuid4()))
email = db.Column(db.String(255), nullable=False)
access_token = db.Column(db.String(255), nullable=False)
refresh_token = db.Column(db.String(255), nullable=False)
expiration_seconds = db.Column(db.Integer, nullable=False)
readwise_api_key = db.Column(db.String(255))
active = db.Column(db.Boolean, default=True)
created_at = db.Column(db.DateTime, default=datetime.utcnow)
updated_at = db.Column(db.DateTime, default=datetime.utcnow)
def __repr__(self):
return f'<Token {self.id}>'
# This table stores email-wise last annotation timestamp
# only one entry per email
class AnnotationLastUpdate(db.Model):
id = db.Column(db.String(36), primary_key=True, default=str(uuid.uuid4()))
email = db.Column(db.String(255), nullable=False)
last_update_time = db.Column(db.DateTime, nullable=False)
created_at = db.Column(db.DateTime, default=datetime.utcnow)
updated_at = db.Column(db.DateTime, default=datetime.utcnow)
def __repr__(self):
return f'<AnnotationLastUpdate {self.id}>'
# Create an application context
with app.app_context():
db.create_all()
# API to create a new token entry
@app.route('/token', methods=['POST'])
def create_token():
data = request.get_json()
email = data.get('email')
access_token = data.get('access_token')
refresh_token = data.get('refresh_token')
expiration_seconds = data.get('expiration_seconds')
readwise_api_key = data.get('readwise_api_key')
required_fields = ['email', 'access_token', 'refresh_token', 'expiration_seconds']
missing_fields = [field for field in required_fields if not data.get(field)]
if missing_fields:
return jsonify({'error': f'Missing required fields: {", ".join(missing_fields)}'}), 400
# unique email when active is true
existing_token = Token.query.filter_by(email=email, active=True).first()
if existing_token:
return jsonify({'error': 'An active token with this email already exists'}), 400
new_token = Token(
email=email,
access_token=access_token,
refresh_token=refresh_token,
expiration_seconds=expiration_seconds,
readwise_api_key=readwise_api_key
)
db.session.add(new_token)
db.session.commit()
return jsonify({'id': new_token.id}), 201
# API to get the token based on the id
@app.route('/token/<id>', methods=['GET'])
def get_token_by_id(id):
token = Token.query.get(id)
if not token:
return jsonify({'error': 'Token not found'}), 404
token_info = {
'id': token.id,
'email': token.email,
'access_token': token.access_token,
'refresh_token': token.refresh_token,
'expiration_seconds': int(token.expiration_seconds),
'readwise_api_key': token.readwise_api_key,
'active': token.active,
'created_at': int(token.created_at.timestamp()),
'updated_at': int(token.updated_at.timestamp())
}
return jsonify({'token': token_info}), 200
# API to get the token based on the email
@app.route('/token', methods=['GET'])
def get_token_by_email():
email = request.args.get('email')
if not email:
return jsonify({'error': 'Missing email query parameter'}), 400
token = Token.query.filter_by(email=email, active=True).first()
if not token:
return '', 204
token_info = {
'id': token.id,
'email': token.email,
'access_token': token.access_token,
'refresh_token': token.refresh_token,
'expiration_seconds': int(token.expiration_seconds),
'readwise_api_key': token.readwise_api_key,
'active': token.active,
'created_at': int(token.created_at.timestamp()),
'updated_at': int(token.updated_at.timestamp())
}
return jsonify({'token': token_info}), 200
# API to update the token based on the id
@app.route('/token/<id>', methods=['PUT'])
def update_token_by_id(id):
token = Token.query.get_or_404(id)
data = request.get_json()
token.access_token = data.get('access_token', token.access_token)
token.refresh_token = data.get('refresh_token', token.refresh_token)
token.expiration_seconds = data.get('expiration_seconds', token.expiration_seconds)
token.readwise_api_key = data.get('readwise_api_key', token.readwise_api_key)
token.updated_at = datetime.utcnow()
db.session.commit()
return '', 204
# deactivate token
@app.route('/token/<id>/deactivate', methods=['POST'])
def deactivate_token_by_id(id):
token = Token.query.get_or_404(id)
token.active = False
db.session.commit()
return '', 204
# get all tokens
@app.route('/token/all', methods=['GET'])
def get_all_tokens():
only_active = request.args.get('only_active')
tokens = Token.query.all() if not only_active else Token.query.filter_by(active=True).all()
tokens_info = [{
'id': token.id,
'email': token.email,
'access_token': token.access_token,
'refresh_token': token.refresh_token,
'expiration_seconds': int(token.expiration_seconds),
'readwise_api_key': token.readwise_api_key,
'active': token.active,
'created_at': int(token.created_at.timestamp()),
'updated_at': int(token.updated_at.timestamp())
} for token in tokens]
return jsonify({'tokens': tokens_info}), 200
# API to create or update the last annotation timestamp
@app.route('/annotation_last_update', methods=['POST'])
def create_or_update_annotation_last_update():
data = request.get_json()
email = data.get('email')
last_update_time = data.get('last_update_time')
required_fields = ['email', 'last_update_time']
missing_fields = [field for field in required_fields if not data.get(field)]
if missing_fields:
return jsonify({'error': f'Missing required fields: {", ".join(missing_fields)}'}), 400
existing_annotation_last_update = AnnotationLastUpdate.query.filter_by(email=email).first()
if existing_annotation_last_update:
existing_annotation_last_update.last_update_time = last_update_time
existing_annotation_last_update.updated_at = datetime.utcnow()
db.session.commit()
return '', 204
else:
new_annotation_last_update = AnnotationLastUpdate(
email=email,
last_update_time=last_update_time
)
db.session.add(new_annotation_last_update)
db.session.commit()
return '', 204
# API to get the last annotation timestamp based on the email
@app.route('/annotation_last_update/<email>', methods=['GET'])
def get_annotation_last_update_by_email(email):
if not email:
return jsonify({'error': 'Missing email query parameter'}), 400
annotation_last_update = AnnotationLastUpdate.query.filter_by(email=email).first()
if not annotation_last_update:
return '', 204
annotation_last_update_info = {
'id': annotation_last_update.id,
'email': annotation_last_update.email,
'last_update_time': int(annotation_last_update.last_update_time.timestamp()),
'created_at': int(annotation_last_update.created_at.timestamp()),
'updated_at': int(annotation_last_update.updated_at.timestamp())
}
return jsonify(annotation_last_update_info), 200
if __name__ == '__main__':
app.run(host='0.0.0.0', port=5000, debug=True)

View File

@ -0,0 +1,2 @@
Flask==3.0.1
Flask-SQLAlchemy==3.1.1

View File

@ -6,6 +6,4 @@ ADD . /app
RUN pip install --no-cache-dir -r requirements.txt
EXPOSE 80
CMD ["python", "main.py"]

3
job/dummy.env Normal file
View File

@ -0,0 +1,3 @@
DATABASE_URL=
INOREADER_CLIENT_ID=
INOREADER_CLIENT_SECRET=

242
job/main.py Normal file
View File

@ -0,0 +1,242 @@
import os
import time
import json
import requests
import logging
DATA_STORE_PATH = "/data/last_update_time.txt"
DATABASE_URL = os.getenv("DATABASE_URL")
logging.basicConfig(level=logging.INFO)
class APIHandler:
def __init__(self, base_url, headers={}):
self.base_url = base_url
self.headers = headers
def get(self, endpoint, params=None):
response = requests.get(self.base_url + endpoint, params=params, headers=self.headers)
response.raise_for_status()
return response.json()
def post(self, endpoint, data=None):
response = requests.post(self.base_url + endpoint, data=json.dumps(data), headers=self.headers)
response.raise_for_status()
return response.status_code
def get_last_update_time(email):
response = requests.get(f'{DATABASE_URL}/annotation_last_update/{email}')
response.raise_for_status()
if response.status_code == 204:
return 0
elif response.status_code == 200:
return response.json()['last_update_time']
def update_last_update_time(email, new_time):
response = requests.post(
f'{DATABASE_URL}/annotation_last_update',
headers={
'Content-Type': 'application/json'
},
json={
'email': email,
'last_update_time': new_time
}
)
response.raise_for_status()
def get_new_annotations(last_annotation_time, inoreader_token):
inoreader = APIHandler(
"https://www.inoreader.com/reader/api/0/stream/contents",
headers = {
'Authorization': 'Bearer ' + inoreader_token()
}
)
all_annotations = []
continuation = None
while True:
params = {
"annotations": 1,
"n": 100,
}
if continuation:
params["c"] = continuation
inoreader_response = inoreader.get(
"/user/-/state/com.google/annotated",
params=params
)
data = json.loads(inoreader_response)
for item in data["items"]:
annotations = item.get("annotations", [])
for annotation in annotations:
annotation['title'] = item['title']
annotation['author'] = item['author']
annotation['sources'] = item['canonical']
all_annotations.append(annotation)
if 'continuation' in data:
continuation = data['continuation']
time.sleep(900) # Sleep for 15 minutes between pages
else:
break
return [annotation for annotation in all_annotations if annotation['added_on'] > last_annotation_time]
def push_annotations_to_readwise(annotations, readwise_token):
readwise = APIHandler(
"https://readwise.io",
headers = {
'Authorization': 'Token ' + readwise_token,
'Content-Type': 'application/json'
}
)
readwise.post(
"/api/v2/highlights/",
data={
'highlights': [
{
'text': annotation['text'],
'title': annotation['title'],
'author': annotation['author'],
'note': annotation['note'],
'highlighted_at': annotation['added_on'],
'category': 'articles',
'source_url': annotation['sources'][0]['href'] if annotation['sources'] else None,
}
for annotation in annotations
]
}
)
# def get_inoreader_access_token():
# response = requests.get(f'{DATABASE_URL}/token/latest')
# response.raise_for_status()
# if response.status_code == 204:
# # throw error - not logged in. Please log in first through the web app
# raise Exception("Not logged in. Please log in first through the web app")
# elif response.status_code == 200:
# resp_json = response.json()
# if resp_json['token']['expiration_seconds'] + resp_json['token']['timestamp'] > datetime.now().timestamp():
# return resp_json['token']['access_token']
# else:
# return refresh_inoreader_access_token(resp_json['token']['refresh_token'])
# access_token = get_token_from_database()
# if not access_token:
# access_token = refresh_inoreader_access_token()
# if not access_token:
# raise Exception("Unable to get access token. Try logging in again through the web app")
# return access_token
def refresh_inoreader_access_token(refresh_token, readwise_api_key):
response = requests.post(
'https://www.inoreader.com/oauth2/token',
headers={
'Content-Type': 'application/x-www-form-urlencoded',
},
data={
'refresh_token': refresh_token,
'client_id': os.getenv("INOREADER_CLIENT_ID"),
'client_secret': os.getenv("INOREADER_CLIENT_SECRET"),
'grant_type': 'refresh_token'
}
)
response.raise_for_status()
token = response.json()
user_info = requests.get('https://www.inoreader.com/reader/api/0/user-info', headers={
'Authorization': f'Bearer {token.get("access_token")}'
}).json()
# Save tokens for later use
save_token(
user_info.get('userEmail'),
token['access_token'],
token['refresh_token'],
token['expires_in'],
readwise_api_key
)
return token['access_token'], readwise_api_key
def save_token(email, access_token, refresh_token, expiration_seconds, readwise_api_key):
response = requests.post(
f'{DATABASE_URL}/token',
headers={
'Content-Type': 'application/json'
},
json={
'email': email,
'access_token': access_token,
'refresh_token': refresh_token,
'expiration_seconds': expiration_seconds,
'readwise_api_key': readwise_api_key
}
)
response.raise_for_status()
def get_all_active_tokens():
response = requests.get(f'{DATABASE_URL}/token/all?only_active=true')
response.raise_for_status()
if response.status_code == 200:
return response.json()['tokens']
else:
return []
def deactivate_token(token_id):
response = requests.post(
f'{DATABASE_URL}/token/{token_id}/deactivate',
headers={
'Content-Type': 'application/json'
}
)
response.raise_for_status()
def check_and_refresh_access_token(token):
if token['expiration_seconds'] + token['timestamp'] > datetime.now().timestamp():
return token['access_token'], token['readwise_api_key']
else:
deactivate_token(token['id'])
return refresh_inoreader_access_token(token['refresh_token'], token['readwise_api_key'])
def main():
while True:
try:
all_tokens = get_all_active_tokens()
for token in all_tokens:
logging.info("Checking for new annotations for user with email: {}".format(token['email']))
inoreader_token, readwise_api_key = check_and_refresh_access_token(token)
last_annotation_time = get_last_update_time(token['email'])
new_annotations = get_new_annotations(last_annotation_time, inoreader_token)
if new_annotations:
latest_added_on = max(annotation['added_on'] for annotation in new_annotations)
push_annotations_to_readwise(new_annotations, readwise_api_key)
update_last_update_time(token['email'], latest_added_on)
logging.info("Successfully pushed {} new annotations to Readwise for user with email: {}".format(len(new_annotations), token['email']))
else:
logging.info("No new annotations found for user with email: {}".format(token['email']))
time.sleep(86400) # Sleep for 24 hours
except Exception as e:
logging.error(f"An error occurred: {e}")
time.sleep(3600) # Sleep for 1 hour in case of error
if __name__ == "__main__":
main()

123
main.py
View File

@ -1,123 +0,0 @@
import os
import time
import json
import requests
import logging
DATA_STORE_PATH = "/data/last_update_time.txt"
logging.basicConfig(level=logging.INFO)
class APIHandler:
def __init__(self, base_url, headers={}):
self.base_url = base_url
self.headers = headers
def get(self, endpoint, params=None):
response = requests.get(self.base_url + endpoint, params=params, headers=self.headers)
response.raise_for_status()
return response.json()
def post(self, endpoint, data=None):
response = requests.post(self.base_url + endpoint, data=json.dumps(data), headers=self.headers)
response.raise_for_status()
return response.status_code
def get_last_update_time():
with open(DATA_STORE_PATH, 'r') as file:
return int(file.read().strip())
def update_last_update_time(new_time):
with open(DATA_STORE_PATH, 'w') as file:
file.write(str(new_time))
def get_new_annotations(last_annotation_time):
inoreader = APIHandler(
"https://www.inoreader.com/reader/api/0/stream/contents",
headers = {
'Authorization': 'Bearer ' + os.getenv("INOREADER_ACCESS_TOKEN")
}
)
all_annotations = []
continuation = None
while True:
params = {
"annotations": 1,
"n": 100,
}
if continuation:
params["c"] = continuation
inoreader_response = inoreader.get(
"/user/-/state/com.google/annotated",
params=params
)
data = json.loads(inoreader_response)
for item in data["items"]:
annotations = item.get("annotations", [])
for annotation in annotations:
annotation['title'] = item['title']
annotation['author'] = item['author']
annotation['sources'] = item['canonical']
all_annotations.append(annotation)
if 'continuation' in data:
continuation = data['continuation']
time.sleep(900) # Sleep for 15 minutes between pages
else:
break
return [annotation for annotation in all_annotations if annotation['added_on'] > last_annotation_time]
def push_annotations_to_readwise(annotations):
readwise = APIHandler(
"https://readwise.io",
headers = {
'Authorization': 'Token ' + os.getenv("READWISE_ACCESS_TOKEN"),
'Content-Type': 'application/json'
}
)
readwise.post(
"/api/v2/highlights/",
data={
'highlights': [
{
'text': annotation['text'],
'title': annotation['title'],
'author': annotation['author'],
'note': annotation['note'],
'highlighted_at': annotation['added_on'],
'category': 'articles',
'source_url': annotation['sources'][0]['href'] if annotation['sources'] else None,
}
for annotation in annotations
]
}
)
def main():
while True:
try:
last_annotation_time = get_last_update_time()
new_annotations = get_new_annotations(last_annotation_time)
if new_annotations:
latest_added_on = max(annotation['added_on'] for annotation in new_annotations)
push_annotations_to_readwise(new_annotations)
update_last_update_time(latest_added_on)
else:
logging.info("No new annotations found")
time.sleep(86400) # Sleep for 24 hours
except Exception as e:
logging.error(f"An error occurred: {e}")
time.sleep(3600) # Sleep for 1 hour in case of error
if __name__ == "__main__":
main()