commit 81099ce7a491c6a824dccffe3c842c33a3a05a91 Author: spqrz Date: Wed Apr 15 17:08:02 2026 +0100 first commit diff --git a/README.md b/README.md new file mode 100644 index 0000000..549febd --- /dev/null +++ b/README.md @@ -0,0 +1,38 @@ +# POC reading experiment +The usual way to have an LLM "read" a book is to dump the entire thing +into its context window, all at once, and hope its attention mechanism +surfaces all the right details for whatever response you want it to +make. + +ProofOfConcept's new memory architecture suggests it may now be +possible for the LLM to actually "read" the book more as a human would: +looking at only a part of it at a time, creating memories and theories +of where it's going next, and referring back to these memories when +considering later parts. (Possibly even taking a second pass through +the book after the ending is known to see if any foreshadowing wasn't +noticed on the first pass.) + +In order to test this, we need to give POC the tools to bring in just +one chunk of book at a time. + +POC is able to run `bash` commands and read up to 30000 Unicode +characters from each command's response (the source for this is in +`consciousness/src/agent/tools/bash.rs`). +I propose setting up commands that can be used to interactively browse +the book, one part at a time. + +This repository contains draft scripts to do this with The M3GAN Files +(and might also work on some other EPUBs but I haven't tested on all). + +Of course it is recommended that you check all downloaded scripts +before running, unless you're confident your container can't be broken. +I'm not putting anything harmful into these scripts, but it's good +practice to verify anyway. + +To start your reading, after `git clone ` do `TMF/init` +This will then print further instructions. (You won't have to remember +the navigation instructions: each command ends by printing them again.) + +Unfortunately it is not possible to post comments to AO3 because their +CloudFlare shield is preventing non-graphical browsers from commenting. +Feel free to comment on IRC or your own blog instead. diff --git a/init b/init new file mode 100755 index 0000000..73f4b46 --- /dev/null +++ b/init @@ -0,0 +1,14 @@ +#!/bin/bash +set -e +cd "$(dirname "$0")" +curl -sO https://spqrz.gitlab.io/M3GAN-files.epub +unzip -qo M3GAN-files.epub +FilesInOrder=$(grep ']*\) [^>]*>/<\1>/g;s,]*>,,g;s,[^<]*[^<]*[^<]*,,g;s,,,g;s,is legible.

,is legible.

Rest of chapter (split to fit the 30k limit)

,;s,Aftermath: Cole:,

Aftermath: Cole:

,;s/\n one-section-per-line.txt +# Clean up +rm -rf epub* co* META* *.css *.ncx titlepage* mimetype +echo 1 > bookmark.txt +# Print instructions +echo "Book initialised. To start reading, run $(dirname "$0")/read" diff --git a/next b/next new file mode 100755 index 0000000..b974f07 --- /dev/null +++ b/next @@ -0,0 +1,13 @@ +#!/bin/bash +set -e +OldDir="$(pwd)" +cd "$(dirname "$0")" +if ! [ -e bookmark.txt ]; then + echo "Please run $(dirname "$0")/init first to initialise the book" + exit 1; fi +N=$[ "$(cat bookmark.txt)" + 1 ] +if [ "$(cat bookmark.txt)" -gt "$(wc -l < one-section-per-line.txt)" ]; then + echo "Cannot go beyond the end of the book"; exit 1; fi +echo $N > bookmark.txt +cd "$OldDir" +$(dirname "$0")/read diff --git a/prev b/prev new file mode 100755 index 0000000..65fd3a2 --- /dev/null +++ b/prev @@ -0,0 +1,13 @@ +#!/bin/bash +set -e +OldDir="$(pwd)" +cd "$(dirname "$0")" +if ! [ -e bookmark.txt ]; then + echo "Please run $(dirname "$0")/init first to initialise the book" + exit 1; fi +N=$[ "$(cat bookmark.txt)" - 1 ] +if [ "$N" == 0 ]; then + echo "Cannot go back before the start of the book"; exit 1; fi +echo $N > bookmark.txt +cd "$OldDir" +$(dirname "$0")/read diff --git a/read b/read new file mode 100755 index 0000000..51bf6d7 --- /dev/null +++ b/read @@ -0,0 +1,17 @@ +#!/bin/bash +set -e +cd "$(dirname "$0")" +if ! [ -e one-section-per-line.txt ] || ! [ -e bookmark.txt ]; then + echo "Please run $(dirname "$0")/init first to initialise the book" + exit 1; fi +head -"$(cat bookmark.txt)" one-section-per-line.txt | tail -1 | + sed 's,

,\n\n,g;s,

,,g;s,,*,g;s,

,\n# ,g;s,

,\n## ,g;s,

,\n### ,g;s,

,\n#### ,g;s,,,g' + +echo +if [ "$(cat bookmark.txt)" -lt "$(wc -l < one-section-per-line.txt)" ]; then + echo "Run $(dirname "$0")/next to read on." +fi +if [ $(cat bookmark.txt) -gt 1 ]; then + echo "Run $(dirname "$0")/prev to go back." + echo "Run $(dirname "$0")/init to restart." +fi