!sudo apt install tesseract-ocr

!ls /usr/share/tesseract-ocr/4.00/tessdata

import os

os.environ["TESSDATA_PREFIX"] = "/usr/share/tesseract-ocr/4.00/tessdata"

# I'm specifying this version number because, as of today (8/31/23), version
# 1.23 is telling me that OCR isn't supported.
!pip install pymupdf==1.22.5

import fitz

print('PyMuPDF version', fitz.VersionBind)
if tuple(map(int, fitz.VersionBind.split("."))) < (1, 19, 1):
    raise ValueError("Need at least v1.19.1 of PyMuPDF")

import time
import datetime

def format_time(elapsed):
    '''
    Takes a time in seconds and returns a string hh:mm:ss
    '''
    # Round to the nearest second.
    elapsed_rounded = int(round((elapsed)))

    # Format as hh:mm:ss
    return str(datetime.timedelta(seconds=elapsed_rounded))

dir = "./drive/MyDrive/Readings/"

import os

files = os.listdir(dir)

# We'll construct a list of just paths to the PDFs.
pdf_filenames = []

# For each file in the directory...
for f in files:

    # Filter for pdfs.
    if '.pdf' in f:

        # Add to the list
        pdf_filenames.append(f)

        print(f)

import os

# For each of the documents...
for (pdf_num, pdf_name) in enumerate(pdf_filenames):


    # Print out which one we're on.
    print('\n======== {:} ({:} of {:}) ========\n'.format(pdf_name, pdf_num + 1, len(pdf_filenames)) )

    # Construct the full path to the file.
    pdf_path = dir + pdf_name

    # Construct the file name for the output by adding the tage " - Parsed" to
    # the end of the filename and replacing the file extension '.pdf' with
    # '.txt'.
    text_file_path = pdf_path[0:-4] + " - Parsed.txt"

    # If the output .txt file already exists, then I'm assuming we already took
    # care of it, so skip this PDF.
    if os.path.exists(text_file_path):
        print('Skipping - Already Parsed.')
        continue

    # Track the time.
    t0 = time.time()

    # ======== Open ========

    # Open the PDF file as a fitz.Document
    doc = fitz.open(pdf_path)


    # ======== Try Text ========
    all_text = ""

    # For each page in the document...
    for i, page in enumerate(doc):

        # Retrieve whatever text exists.
        all_text += page.get_text()

    # If there wasn't any text...
    if len(all_text) == 0:

        # ======== Run OCR =========
        # We'll try parsing the images.

        print('  Running OCR...')

        # For each page...
        for i, page in enumerate(doc):

            # This can be slow, so we'll print out the current page number to
            # show our progress.
            print('    Page {:} of {:}'.format(i + 1, len(doc)))

            # Make the `TextPage` object using the `get_textpage_ocr` function.
            # This is the step that does all of the OCR.
            full_tp = page.get_textpage_ocr(flags=0, dpi=300, full=True)

            # Next, we'll pass this `full_tp` to the `get_text` function to
            # pull out the text.
            #
            # By passing the "blocks" argument, we'll get a list of text blocks
            # from the page. I'm guessing this means the library first
            # identifies regions of text in the image, and then runs OCR on
            # them separately.
            #
            # Some example code from the library's GitHub repo showed this
            # approach, where we are eliminating *all* newline characters from
            # the text. I followed this example because, wihtout it, there does
            # seem to be a ton of whitespace! ChatGPT won't care, but if you
            # want to read the parsed file yourself, it helps to clean that up.

            # Get the parsed text blocks.
            blocks = page.get_text("blocks", textpage=full_tp)

            # For each block...
            for b in blocks:
                # The first four elements of 'b' appear to be something like
                # coordinates, and the fifth element, b[4], is the actual text.
                #
                # Replace *all* of the newline characters with a single space,
                # but then add back in a single newline character at the end
                # to separate the blocks.
                all_text += b[4].replace("\n", " ") + '\n'

            # Alternatively, if you ommit the "blocks" argument, you'll just
            # get a single string with all of the text.
            #all_text += page.get_text(textpage=full_tp)

    # ======== Record to Disk ========
    print('  Writing out scanned text to:\n    ', text_file_path)

    # Write all of the text to the .txt file.
    with open(text_file_path, "w") as f:
        f.write(all_text)

    print('  Done.')

    print('  Elapsed:', format_time(time.time() - t0))

!pip install openai

from google.colab import userdata

api_key = userdata.get('Chris_API_key')

import openai

# Set up the OpenAI API client
openai.api_key = api_key

import openai

prompt = "Tell me a joke."

# Send our message to ChatGPT
completion = openai.chat.completions.create(
    model="gpt-3.5-turbo",
    messages=[
        {
            "role": "user",
            "content": prompt,
        },
    ],
)

# Extract the reply text.
reply = completion.choices[0].message.content

# Print the reply
print(reply)

%pip install --upgrade tiktoken

import tiktoken

# Use tiktoken.encoding_for_model() to automatically load the correct tokenizer
# for a given model name.
encoder = tiktoken.encoding_for_model("gpt-3.5-turbo")

text = "tiktoken is great!"

# Turn text into tokens with encoding.encode(), which converts the string into a
# list of integers.
token_ids = encoder.encode(text)

print(text, "-->")
print(token_ids)

# It will print: [83, 1609, 5963, 374, 2294, 0]

# The length of the list tells us how many tokens are required to represent our
# text.
num_tokens = len(token_ids)

print("\nThe text string breaks into", num_tokens, "tokens.")

# Convert back from token numbers into a string.
# Note that the 'decode' function is able to exactly reproduce the
# input string--nothing is lost from encoding and decoding.
str_chunk = encoder.decode(token_ids)

print("The reconstructed string:")
print(str_chunk)

# How many tokens in each chunk.
chunk_size = 3000

import os

txt_filenames = []

#dir = "./drive/MyDrive/Readings/"

files = os.listdir(dir)

for f in files:

    # Filter for pdfs.
    if ' - Parsed.txt' in f:

        # Add to the list
        txt_filenames.append(f)

        print(f)

import os
import numpy as np
import textwrap

# This tiktoken 'Encoding' object can tokenize and de-tokenize our text.
encoding = tiktoken.get_encoding("cl100k_base")

# When we print out the final summary, we'll wrap text to 80 characters.
wrapper = textwrap.TextWrapper(width=80)

# For each of the documents...
for (txt_num, txt_name) in enumerate(txt_filenames):

    # We already filtered for this, but just to sanity check, ensure we're only
    # running this on the "Parsed" files, and not the summaries. :-P
    assert(" - Parsed.txt" in txt_name)

    # Get the base filename.
    base_name = txt_name[0:-len(" - Parsed.txt")]

    # We'll be writing out two more files--the final summary, and a log of
    # the actual chat. (Note: I ommit the article chunks)
    summary_name = base_name + " - Summary.txt"
    log_name = base_name + " - Chat Log.txt"

    # Print the name of the article we're about to work on.
    print('\n======== {:} ({:} of {:}) ========\n'.format(base_name, txt_num + 1, len(txt_filenames)) )

    # If the summary output already exists...
    if os.path.exists(dir + summary_name):
         print('  Skipping - Already Summarized.')
         continue

    # Track the time.
    t0 = time.time()

    # Read in the full text of the article.
    with open(dir + txt_name, 'r') as f:
        lines = f.readlines()

    # Combine all of the lines into a single string.
    text = "".join(lines)

    # ======== Tokenize ========

    # Using tiktoken, we can split our string into ChatGPT tokens. Each token
    # is represented here by its index in the vocabulary.
    tokens = encoding.encode(text)

    # Determine the number of chunks we'll need to break this into.
    chunk_count = float(len(tokens)) / float(chunk_size)

    # Round up.
    chunk_count = int(np.ceil(chunk_count))

    # Print out the full token count of the article, and how many chunks we'll
    # be breaking it into.
    print("  Token count: {:,}   ({:} chunks of {:,} tokens)".format(len(tokens), chunk_count, chunk_size))

    # ======== Split Chunks ========

    str_chunks = []

    # For each chunk of tokens...
    for i in range(0, len(tokens), chunk_size):

        # Get the next cunk of tokens.
        # (Note that Python is nice enough to automatically handle the end of
        # the array for us.)
        token_chunk = tokens[i:i + chunk_size]

        # Convert back from token numbers into a string.
        # Note that the 'decode' function is able to exactly reproduce the
        # input string--nothing is lost from encoding and decoding.
        str_chunk = encoding.decode(token_chunk)

        # If the chunks line up such that the last one is tiny, then skip it.
        # I figure too small of an input might do more harm than good to the
        # summary--but I haven't tested that theory!
        if len(str_chunk) < 1000:
            print('Dropping the last chunk -- only {:} characters'.format(len(str_chunk)))
            continue

        str_chunks.append(str_chunk)

    num_chunks = len(str_chunks)

    # ======== Summarize! ========
    with open(dir + log_name, 'w') as f:

        print('\nSummarizing chunks...')

        chunk_summaries = []
        agg_summary = ""
        acc_prompt = ""

        # For each of the chunks...
        for i in range(len(str_chunks)):

            print('\n  Chunk {:} of {:}'.format(i + 1, num_chunks))

            # ======== Step 1: Summarize Next Chunk ========
            # Provide a summary of the past chunks, plus the new chunk.
            # Ask ChatGPT to create a summary of the new chunk.

            # The first chunk is a special case since we don't have context to
            # provide yet.
            if i == 0:
                # State the problem and what we want.
                prompt = \
"I am working on creating a summary of an article which I have broken into {:} \
segments. Below is the first segment; please write a bullet point summary of \
it in 250 words or less:\n\n".format(num_chunks) + str_chunks[0]

            # For all subsequent chunks, we'll provide the summary plus the new
            # chunk.
            else:
                # State the problem.
                prompt = \
"I am working on creating a summary of a long article or book chapter which I \
have broken into {:} segments.\nFor context, here is a summary of the first \
{:} segments:\n\n".format(num_chunks, i)

                # Insert the summary of the prior chunks.
                prompt += agg_summary + "\n"

                # Ask it to summarize.
                prompt += \
"\nPlease write a bullet point summary of this next segment in 250 words \
or less:\n\n"
                # Add the text for the current chunk.
                prompt += str_chunks[i]

            # Send it to ChatGPT!
            completion = openai.chat.completions.create(
                model="gpt-3.5-turbo",
                messages=[
                    {
                        "role": "user",
                        "content": prompt,
                    },
                ],
            )

            # Extract the reply text.
            reply = completion.choices[0].message.content

            # Update the list of summaries.
            chunk_summaries.append(reply)

            # Write the response to the chat log file.
            f.write("\n\n▂▂▂▂▂▂▂▂▂▂▂▂▂ ↓ Prompt (Chunk {:} of {:}) ↓ ▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂\n".format(i + 1, num_chunks))
            f.write(prompt)
            f.write("\n▂▂▂▂▂▂▂▂▂▂▂▂▂ ↓ Reply (Chunk {:} of {:}) ↓ ▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂\n".format(i + 1, num_chunks))
            f.write(reply)

            # Report token counts.
            num_prompt_tokens = len(encoder.encode(prompt))
            num_reply_tokens = len(encoder.encode(reply))
            total_tokens = num_prompt_tokens + num_reply_tokens

            print('    Token counts - Prompt: {:,}    Reply: {:,}    Total: {:,}  (max is 4,096)'.format(num_prompt_tokens, num_reply_tokens, total_tokens))

            # ======== Step 2: Summarize the Summaries ========
            # Ask ChatGPT to consolidate all of the existing chunk summaries
            # down into a single condensed version.

            # For the first chunk, we don't need to ask it to consolidate
            # anything.
            if i == 0:
                agg_summary = reply

            # For subsequent chunks, provide each of the separate summaries and
            # ask it to create a single, condensed version.
            else:
                # Explain the task, and what we want.
                prompt = \
"I am working on creating a summary of a long article or book chapter which I \
have broken into {:} segments.\n Please consolidate the following summaries of \
the first {:} segments down into a single bullet point summary that is 250 \
words or less:\n".format(num_chunks, i+1)

                # Add each of the separate summaries:
                for (s_i, s) in enumerate(chunk_summaries):
                    prompt += "\nSummary of Segment {:}:\n".format(s_i + 1)
                    prompt += s

                # Send it to ChatGPT!
                completion = openai.chat.completions.create(
                    model="gpt-3.5-turbo",
                    messages=[
                        {
                            "role": "user",
                            "content": prompt,
                        },
                    ],
                )

                # Extract the reply text.
                reply = completion.choices[0].message.content

                # The reply is our new consolidated summary of the prior chunks.
                agg_summary = reply

                # Write the response to the chat log file.
                f.write("\n\n▂▂▂▂▂▂▂▂▂▂▂▂▂ ↓ Prompt (Chunk {:} of {:}) ↓ ▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂\n".format(i + 1, len(str_chunks)))
                f.write(prompt)
                f.write("\n▂▂▂▂▂▂▂▂▂▂▂▂▂ ↓ Reply (Chunk {:} of {:}) ↓ ▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂\n".format(i + 1, len(str_chunks)))
                f.write(reply)

            # End of the chunk loop!

    # At this point, we have summaries of all of the segments, and a single,
    # final summary.

    # Print the final summary! This uses the wrapper tool (instantiated at the
    # top of this cell) to wrap the text to 80-characters wide, for readability.
    print('\nFinal, overall summary:\n')
    print('--------')
    print(wrapper.fill(agg_summary))
    print('--------')
    print('')

    # ======== Glossary ========
    # As a final step, let's ask for a glossary of key terms!

    # Explain the task and what we want.
    prompt = \
"The following is a summary of a chapter of a book. Can you create a glossary \
for the key terms that readers might not be familiar with?\n\n" + agg_summary

    # Send it to chat gpt!
    completion = openai.chat.completions.create(
        model="gpt-3.5-turbo",
        messages=[
            {
                "role": "user",
                "content": prompt,
            },
        ],
    )

    # Extract the reply text.
    reply = completion.choices[0].message.content

    # The reply should be a glossary of key terms.
    glossary = reply

    # Save the final summary to a text file.
    with open(dir + summary_name, 'w') as f:

        f.write("This document contains:\n")
        f.write("  1. The overall summary of the document.\n")
        f.write("  2. A glossary of key terms.\n")
        f.write("  3. The separate summaries of each chunk of text that the doc was broken into.\n")
        f.write("\n")
        f.write("▂▂▂▂▂▂▂ Overall Summary ▂▂▂▂▂▂▂\n")
        f.write(agg_summary)

        f.write("\n\n")
        f.write("▂▂▂▂▂▂▂ Glossary of Key Terms ▂▂▂▂▂▂▂\n")
        f.write(glossary)

        # For each separate chunk summary...
        for (s_i, s) in enumerate(chunk_summaries):
            f.write("\n\n")
            f.write("▂▂▂▂▂▂▂ Summary of Chunk {:} of {:} ▂▂▂▂▂▂▂\n".format(s_i + 1, num_chunks))
            f.write(s)

dir = './drive/MyDrive/Readings/Split-PDFs/'

filename = 'Machiavelli - The Prince.pdf'

# Remove the '.pdf' extension to get the base name.
base_name = filename[0:-4]

# This divide the PDF into 10 separate PDFs. The first will contain pages 1-22,
# the second will contain pages 23-51, and so on.
#split_starts = [1, 23, 52, 73, 98, 123, 139, 163, 192, 205]

# Pg 127 is the start of chapter 18, pg 130 is chapter 19, pg 140 is chapter 20.
split_starts = [1, 127, 130, 140]

final_page = -1

# Map, e.g., from a book's table of contents to page numbers within the PDF.
# In the below example, Chapter 1 of the book is on page 1 of the book, but page
# 23 of the PDF. Chapter is on pg. 30 of the book, but page 52 of the PDF. And
# so on.
if False:
    # Page numbers according to the book.
    book_pages = [1, 30, 51, 76, 101, 117, 141, 170, 183]

    # Page one of the book occurs on page 23 of the PDF, so the offset is 22.
    pdf_offset = 22

    # The first split will contain pages 1-22 of the PDF.
    pdf_splits = [1]

    # Add the PDF offset to each of the book page numbers.
    for bp in book_pages:
        pdf_pages.append(bp + pdf_offset)

    split_starts = pdf_pages

    print(pdf_pages)

import sys, fitz

with fitz.open(dir + filename) as src_doc:

    print('Splitting', dir + filename)

    # Set the ending page of the last split to be the last page of the PDF, if
    # not specified.
    if final_page == -1:
        final_page = len(src_doc)
        print('Final page =', final_page)

    # For each of the splits...
    for (split_i, start_page) in enumerate(split_starts):

        # For the purposes of printing progress and naming the files, number the
        # splits starting from 1.
        split_num = split_i + 1
        print('  Split', split_num)

        # Determine what should be the last page of the current split.

        # If it's the final split, use the 'final_page' variable.
        if split_num == len(split_starts):
            end_page = final_page

        # Otherwise, use one less than the start of the next split.
        else:
            end_page = split_starts[split_i + 1] - 1

        # Create a new PDF object for this split.
        split_doc = fitz.open()

        # Add the pages from the current split.
        split_doc.insert_pdf(
            src_doc, # The original doc
            from_page = start_page - 1, # 0-indexed, so subtract 1.
            to_page = end_page - 1,
        )

        # Write out the new PDF, adding in the split number.
        split_doc.save(dir + base_name + ' - {:}.pdf'.format(split_num))

        split_doc.close()

import fitz  # PyMuPDF

input_pdf_path = "./drive/MyDrive/Readings/Example Article.pdf"
output_pdf_path = "./drive/MyDrive/Readings/Example Article - Rotated.pdf"

# Open the PDF file
pdf_document = fitz.open(input_pdf_path)

# Iterate through each page and rotate it 90 degrees counter-clockwise
for page_num in range(pdf_document.page_count):
    page = pdf_document[page_num]
    page.set_rotation(-90)  # Rotate 90 degrees counter-clockwise

# Save the modified PDF to a new file
pdf_document.save(output_pdf_path)
pdf_document.close()

!pip install beautifulsoup4

import os

dir = "./drive/MyDrive/Readings/"

files = os.listdir(dir)

# We'll construct a list of just paths to the HTML files.
html_filenames = []

# For each file in the directory...
for f in files:

    # Filter for HTML.
    if '.html' in f:

        # Add to the list
        html_filenames.append(f)

        print(f)

from bs4 import BeautifulSoup

# For each of the documents...
for (html_num, html_name) in enumerate(html_filenames):

    # Print out which one we're on.
    print('\n======== {:} ({:} of {:}) ========\n'.format(html_name, html_num + 1, len(html_filenames)) )

    # Construct the full path to the file.
    html_path = dir + html_name

    # Construct the file name for the output by adding the tage " - Parsed" to
    # the end of the filename and replacing the file extension '.html' with
    # '.txt'.
    text_file_path = html_path[0:-5] + " - Parsed.txt"

    with open(dir + html_name, 'r') as f:
        # Read the HTML file.
        html_content = f.readlines()

        # Convert from a list to a single string.
        html_content = '\n'.join(html_content)

        # Set up the HTML parser.
        soup = BeautifulSoup(html_content, 'html.parser')

        # Estract the plain text.
        plain_text = soup.get_text()

        # Write all of the text to the .txt file.
        with open(text_file_path, "w") as f:
            f.write(all_text)

# Example HTML content
html_content = """
<html>
    <head>
        <title>Sample HTML</title>
    </head>
    <body>
        <h1>Hello, World!</h1>
        <p>This is a sample HTML document.</p>
        <ul>
            <li>Item 1</li>
            <li>Item 2</li>
            <li>Item 3</li>
        </ul>
    </body>
</html>
"""