From 5a5f737ff42ed225f69b551b3f7331337af3f8d9 Mon Sep 17 00:00:00 2001
From: Swapnil <swapnil.theskss@gmail.com>
Date: Wed, 17 Jan 2024 19:20:22 +0530
Subject: [PATCH] incorporate suggestions: - fix get_last_update_time (int) -
 use continuation query - sleep 15 min b/w pages - sleep 24 hrs b/w each cycle
 - sleep 1 hr if error - Push extra highlights properties

---
 main.py | 51 ++++++++++++++++++++++++++++++++++-----------------
 1 file changed, 34 insertions(+), 17 deletions(-)

diff --git a/main.py b/main.py
index 980dec4..656f0cd 100644
--- a/main.py
+++ b/main.py
@@ -25,11 +25,11 @@ class APIHandler:
 
 def get_last_update_time():
     with open(DATA_STORE_PATH, 'r') as file:
-        return file.read().strip()
+        return int(file.read().strip())
 
 def update_last_update_time(new_time):
     with open(DATA_STORE_PATH, 'w') as file:
-        file.write(new_time)
+        file.write(str(new_time))
 
 def get_new_annotations(last_annotation_time):
     inoreader = APIHandler(
@@ -39,22 +39,36 @@ def get_new_annotations(last_annotation_time):
         }
     )
 
-    inoreader_response = inoreader.get(
-        "/user/-/state/com.google/annotated",
-        params={
+    all_annotations = []
+    continuation = None
+
+    while True:
+        params = {
             "annotations": 1,
             "n": 100,
         }
-    )
-    data = json.loads(inoreader_response)
-    all_annotations = []
+        if continuation:
+            params["c"] = continuation
 
-    for item in data["items"]:
-        annotations = item.get("annotations", [])
-        for annotation in annotations:
-            annotation['title'] = item['title']
-            annotation['author'] = item['author']
-            all_annotations.append(annotation)
+        inoreader_response = inoreader.get(
+            "/user/-/state/com.google/annotated",
+            params=params
+        )
+        data = json.loads(inoreader_response)
+
+        for item in data["items"]:
+            annotations = item.get("annotations", [])
+            for annotation in annotations:
+                annotation['title'] = item['title']
+                annotation['author'] = item['author']
+                annotation['sources'] = item['canonical']
+                all_annotations.append(annotation)
+
+        if 'continuation' in data:
+            continuation = data['continuation']
+            time.sleep(900)  # Sleep for 15 minutes between pages
+        else:
+            break
 
     return [annotation for annotation in all_annotations if annotation['added_on'] > last_annotation_time]
 
@@ -76,6 +90,9 @@ def push_annotations_to_readwise(annotations):
                     'title': annotation['title'],
                     'author': annotation['author'],
                     'note': annotation['note'],
+                    'highlighted_at': annotation['added_on'],
+                    'category': 'articles',
+                    'source_url': annotation['sources'][0]['href'] if annotation['sources'] else None,
                 }
                 for annotation in annotations
             ]
@@ -96,11 +113,11 @@ def main():
             else:
                 logging.info("No new annotations found")
 
-            time.sleep(3600)
+            time.sleep(86400)  # Sleep for 24 hours
 
         except Exception as e:
             logging.error(f"An error occurred: {e}")
-            time.sleep(60)
+            time.sleep(3600)  # Sleep for 1 hour in case of error
 
 if __name__ == "__main__":
-    main()
+    main()
\ No newline at end of file