From 5a5f737ff42ed225f69b551b3f7331337af3f8d9 Mon Sep 17 00:00:00 2001 From: Swapnil Date: Wed, 17 Jan 2024 19:20:22 +0530 Subject: [PATCH] incorporate suggestions: - fix get_last_update_time (int) - use continuation query - sleep 15 min b/w pages - sleep 24 hrs b/w each cycle - sleep 1 hr if error - Push extra highlights properties --- main.py | 51 ++++++++++++++++++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 17 deletions(-) diff --git a/main.py b/main.py index 980dec4..656f0cd 100644 --- a/main.py +++ b/main.py @@ -25,11 +25,11 @@ class APIHandler: def get_last_update_time(): with open(DATA_STORE_PATH, 'r') as file: - return file.read().strip() + return int(file.read().strip()) def update_last_update_time(new_time): with open(DATA_STORE_PATH, 'w') as file: - file.write(new_time) + file.write(str(new_time)) def get_new_annotations(last_annotation_time): inoreader = APIHandler( @@ -39,22 +39,36 @@ def get_new_annotations(last_annotation_time): } ) - inoreader_response = inoreader.get( - "/user/-/state/com.google/annotated", - params={ + all_annotations = [] + continuation = None + + while True: + params = { "annotations": 1, "n": 100, } - ) - data = json.loads(inoreader_response) - all_annotations = [] + if continuation: + params["c"] = continuation - for item in data["items"]: - annotations = item.get("annotations", []) - for annotation in annotations: - annotation['title'] = item['title'] - annotation['author'] = item['author'] - all_annotations.append(annotation) + inoreader_response = inoreader.get( + "/user/-/state/com.google/annotated", + params=params + ) + data = json.loads(inoreader_response) + + for item in data["items"]: + annotations = item.get("annotations", []) + for annotation in annotations: + annotation['title'] = item['title'] + annotation['author'] = item['author'] + annotation['sources'] = item['canonical'] + all_annotations.append(annotation) + + if 'continuation' in data: + continuation = data['continuation'] + time.sleep(900) # Sleep for 15 minutes between pages + else: + break return [annotation for annotation in all_annotations if annotation['added_on'] > last_annotation_time] @@ -76,6 +90,9 @@ def push_annotations_to_readwise(annotations): 'title': annotation['title'], 'author': annotation['author'], 'note': annotation['note'], + 'highlighted_at': annotation['added_on'], + 'category': 'articles', + 'source_url': annotation['sources'][0]['href'] if annotation['sources'] else None, } for annotation in annotations ] @@ -96,11 +113,11 @@ def main(): else: logging.info("No new annotations found") - time.sleep(3600) + time.sleep(86400) # Sleep for 24 hours except Exception as e: logging.error(f"An error occurred: {e}") - time.sleep(60) + time.sleep(3600) # Sleep for 1 hour in case of error if __name__ == "__main__": - main() + main() \ No newline at end of file