Merge branch 'main' of github.com:danielmiessler/fabric

This commit is contained in:
Daniel Miessler 2024-11-08 21:19:17 -08:00
commit 73a0e38af6
13 changed files with 465 additions and 69 deletions

6
.dockerignore Normal file
View file

@ -0,0 +1,6 @@
.git
.gitignore
.env
README.md
docker-compose.yml
Dockerfile

View file

@ -41,7 +41,10 @@ jobs:
minor=$(echo "$latest_tag" | cut -d. -f2)
patch=$(echo "$latest_tag" | cut -d. -f3)
new_patch=$((patch + 1))
new_tag="v${major}.${minor}.${new_patch}"
new_version="${major}.${minor}.${new_patch}"
new_tag="v${new_version}"
echo "New version is: $new_version"
echo "new_version=$new_version" >> $GITHUB_ENV # Save the new version to environment file
echo "New tag is: $new_tag"
echo "new_tag=$new_tag" >> $GITHUB_ENV # Save the new tag to environment file
@ -53,11 +56,12 @@ jobs:
- name: Update version.nix file
run: |
echo "\"${{ env.new_tag }}\"" > pkgs/fabric/version.nix
echo "\"${{ env.new_version }}\"" > pkgs/fabric/version.nix
- name: Commit changes
run: |
git add version.go
git add pkgs/fabric/version.nix
if ! git diff --staged --quiet; then
git commit -m "Update version to ${{ env.new_tag }} and commit $commit_hash"
else

41
Dockerfile Normal file
View file

@ -0,0 +1,41 @@
# Use official golang image as builder
FROM golang:1.22.5-alpine AS builder
# Set working directory
WORKDIR /app
# Copy go mod and sum files
COPY go.mod go.sum ./
# Download dependencies
RUN go mod download
# Copy source code
COPY . .
# Build the application
RUN CGO_ENABLED=0 GOOS=linux go build -o fabric
# Use scratch as final base image
FROM alpine:latest
# Copy the binary from builder
COPY --from=builder /app/fabric /fabric
# Copy patterns directory
COPY patterns /patterns
# Ensure clean config directory and copy ENV file
RUN rm -rf /root/.config/fabric && \
mkdir -p /root/.config/fabric
COPY ENV /root/.config/fabric/.env
# Add debug commands
RUN ls -la /root/.config/fabric/
# Expose port 8080
EXPOSE 8080
# Run the binary with debug output
ENTRYPOINT ["/fabric"]
CMD ["--serve"]

9
ENV Normal file
View file

@ -0,0 +1,9 @@
DEFAULT_VENDOR=OpenRouter
DEFAULT_MODEL=openai/gpt-3.5-turbo-0125
DEFAULT_MODEL_CONTEXT_LENGTH=128K
PATTERNS_LOADER_GIT_REPO_URL=https://github.com/danielmiessler/fabric.git
PATTERNS_LOADER_GIT_REPO_PATTERNS_FOLDER=patterns
OPENROUTER_API_KEY=sk-or-v1-
OPENROUTER_API_BASE_URL=https://openrouter.ai/api/v1
YOUTUBE_API_KEY=AIzaS
JINA_AI_API_KEY=jina_57

108
README.md
View file

@ -29,23 +29,34 @@
## Navigation
- [Updates](#updates)
- [What and Why](#what-and-why)
- [Philosophy](#philosophy)
- [Breaking problems into components](#breaking-problems-into-components)
- [Too many prompts](#too-many-prompts)
- [The Fabric approach to prompting](#our-approach-to-prompting)
- [Installation](#Installation)
- [Migration](#Migration)
- [Upgrading](#Upgrading)
- [Usage](#Usage)
- [Examples](#examples)
- [`fabric`](#fabric)
- [Navigation](#navigation)
- [Updates](#updates)
- [Intro videos](#intro-videos)
- [What and why](#what-and-why)
- [Philosophy](#philosophy)
- [Breaking problems into components](#breaking-problems-into-components)
- [Too many prompts](#too-many-prompts)
- [Installation](#installation)
- [Get Latest Release Binaries](#get-latest-release-binaries)
- [From Source](#from-source)
- [Environment Variables](#environment-variables)
- [Setup](#setup)
- [Add aliases for all patterns](#add-aliases-for-all-patterns)
- [Save your files in markdown using aliases](#save-your-files-in-markdown-using-aliases)
- [Migration](#migration)
- [Upgrading](#upgrading)
- [Usage](#usage)
- [Our approach to prompting](#our-approach-to-prompting)
- [Examples](#examples)
- [Just use the Patterns](#just-use-the-patterns)
- [Custom Patterns](#custom-patterns)
- [Helper Apps](#helper-apps)
- [pbpaste](#pbpaste)
- [Meta](#meta)
- [Primary contributors](#primary-contributors)
- [Custom Patterns](#custom-patterns)
- [Helper Apps](#helper-apps)
- [`to_pdf`](#to_pdf)
- [`to_pdf` Installation](#to_pdf-installation)
- [pbpaste](#pbpaste)
- [Meta](#meta)
- [Primary contributors](#primary-contributors)
<br />
@ -168,6 +179,69 @@ fabric --setup
```
If everything works you are good to go.
### Add aliases for all patterns
In order to add aliases for all your patterns and use them directly as commands ie. `summarize` instead of `fabric --pattern summarize`
You can add the following to your `.zshrc` or `.bashrc` file.
```bash
# Loop through all files in the ~/.config/fabric/patterns directory
for pattern_file in $HOME/.config/fabric/patterns/*; do
# Get the base name of the file (i.e., remove the directory path)
pattern_name=$(basename "$pattern_file")
# Create an alias in the form: alias pattern_name="fabric --pattern pattern_name"
alias_command="alias $pattern_name='fabric --pattern $pattern_name'"
# Evaluate the alias command to add it to the current shell
eval "$alias_command"
done
yt() {
local video_link="$1"
fabric -y "$video_link" --transcript
}
```
This also creates a `yt` alias that allows you to use `yt https://www.youtube.com/watch?v=4b0iet22VIk` to get your transcripts.
#### Save your files in markdown using aliases
If in addition to the above aliases you would like to have the option to save the output to your favourite markdown note vault like Obsidian then instead of the above add the following to your `.zshrc` or `.bashrc` file:
```bash
# Define the base directory for Obsidian notes
obsidian_base="/path/to/obsidian"
# Loop through all files in the ~/.config/fabric/patterns directory
for pattern_file in ~/.config/fabric/patterns/*; do
# Get the base name of the file (i.e., remove the directory path)
pattern_name=$(basename "$pattern_file")
# Define a function dynamically for each pattern
eval "
$pattern_name() {
local title=\$1
local date_stamp=\$(date +'%Y-%m-%d')
local output_path=\"\$obsidian_base/\${date_stamp}-\${title}.md\"
# Check if a title was provided
if [ -n \"\$title\" ]; then
# If a title is provided, use the output path
fabric --pattern \"$pattern_name\" -o \"\$output_path\"
else
# If no title is provided, use --stream
fabric --pattern \"$pattern_name\" --stream
fi
}
"
done
yt() {
local video_link="$1"
fabric -y "$video_link" --transcript
}
```
This will allow you to use the patterns as aliases like in the above for example `summarize` instead of `fabric --pattern summarize --stream`, however if you pass in an extra argument like this `summarize "my_article_title"` your output will be saved in the destination that you set in `obsidian_base="/path/to/obsidian"` in the following format `YYYY-MM-DD-my_article_title.md` where the date gets autogenerated for you.
You can tweak the date format by tweaking the `date_stamp` format.
### Migration
@ -353,7 +427,7 @@ This will create a PDF file named `output.pdf` in the current directory.
To install `to_pdf`, install it the same way as you install Fabric, just with a different repo name.
```bash
go install github.com/danielmiessler/fabric/to_pdf@latest
go install github.com/danielmiessler/fabric/plugins/tools/to_pdf@latest
```
Make sure you have a LaTeX distribution (like TeX Live or MiKTeX) installed on your system, as `to_pdf` requires `pdflatex` to be available in your system's PATH.

View file

@ -2,6 +2,7 @@ package cli
import (
"fmt"
"github.com/danielmiessler/fabric/plugins/tools/youtube"
"os"
"path/filepath"
"strconv"
@ -144,37 +145,38 @@ func Cli(version string) (err error) {
}
var videoId string
if videoId, err = registry.YouTube.GetVideoId(currentFlags.YouTube); err != nil {
var playlistId string
if videoId, playlistId, err = registry.YouTube.GetVideoOrPlaylistId(currentFlags.YouTube); err != nil {
return
} else if (videoId == "" || currentFlags.YouTubePlaylist) && playlistId != "" {
if currentFlags.Output != "" {
err = registry.YouTube.FetchAndSavePlaylist(playlistId, currentFlags.Output)
} else {
var videos []*youtube.VideoMeta
if videos, err = registry.YouTube.FetchPlaylistVideos(playlistId); err != nil {
err = fmt.Errorf("error fetching playlist videos: %v", err)
return
}
for _, video := range videos {
var message string
if message, err = processYoutubeVideo(currentFlags, registry, video.Id); err != nil {
return
}
if !currentFlags.IsChatRequest() {
if err = WriteOutput(message, fmt.Sprintf("%v.md", video.TitleNormalized)); err != nil {
return
}
} else {
messageTools = AppendMessage(messageTools, message)
}
}
}
return
}
if !currentFlags.YouTubeComments || currentFlags.YouTubeTranscript {
var transcript string
var language = "en"
if currentFlags.Language != "" || registry.Language.DefaultLanguage.Value != "" {
if currentFlags.Language != "" {
language = currentFlags.Language
} else {
language = registry.Language.DefaultLanguage.Value
}
}
if transcript, err = registry.YouTube.GrabTranscript(videoId, language); err != nil {
return
}
messageTools = AppendMessage(messageTools, transcript)
}
if currentFlags.YouTubeComments {
var comments []string
if comments, err = registry.YouTube.GrabComments(videoId); err != nil {
return
}
commentsString := strings.Join(comments, "\n")
messageTools = AppendMessage(messageTools, commentsString)
}
messageTools, err = processYoutubeVideo(currentFlags, registry, videoId)
if !currentFlags.IsChatRequest() {
err = currentFlags.WriteOutput(messageTools)
return
@ -254,3 +256,43 @@ func Cli(version string) (err error) {
}
return
}
func processYoutubeVideo(
flags *Flags, registry *core.PluginRegistry, videoId string) (message string, err error) {
if !flags.YouTubeComments || flags.YouTubeTranscript {
var transcript string
var language = "en"
if flags.Language != "" || registry.Language.DefaultLanguage.Value != "" {
if flags.Language != "" {
language = flags.Language
} else {
language = registry.Language.DefaultLanguage.Value
}
}
if transcript, err = registry.YouTube.GrabTranscript(videoId, language); err != nil {
return
}
message = AppendMessage(message, transcript)
}
if flags.YouTubeComments {
var comments []string
if comments, err = registry.YouTube.GrabComments(videoId); err != nil {
return
}
commentsString := strings.Join(comments, "\n")
message = AppendMessage(message, commentsString)
}
return
}
func WriteOutput(message string, outputFile string) (err error) {
fmt.Println(message)
if outputFile != "" {
err = CreateOutputFile(message, outputFile)
}
return
}

View file

@ -42,7 +42,8 @@ type Flags struct {
OutputSession bool `long:"output-session" description:"Output the entire session (also a temporary one) to the output file"`
LatestPatterns string `short:"n" long:"latest" description:"Number of latest patterns to list" default:"0"`
ChangeDefaultModel bool `short:"d" long:"changeDefaultModel" description:"Change default model"`
YouTube string `short:"y" long:"youtube" description:"YouTube video \"URL\" to grab transcript, comments from it and send to chat"`
YouTube string `short:"y" long:"youtube" description:"YouTube video or play list \"URL\" to grab transcript, comments from it and send to chat or print it put to the console and store it in the output file"`
YouTubePlaylist bool `long:"playlist" description:"Prefer playlist over video if both ids are present in the URL"`
YouTubeTranscript bool `long:"transcript" description:"Grab transcript from YouTube video and send to chat (it used per default)."`
YouTubeComments bool `long:"comments" description:"Grab comments from YouTube video and send to chat"`
Language string `short:"g" long:"language" description:"Specify the Language Code for the chat, e.g. -g=en -g=zh" default:""`
@ -72,16 +73,19 @@ func Init() (ret *Flags, err error) {
}
info, _ := os.Stdin.Stat()
hasStdin := (info.Mode() & os.ModeCharDevice) == 0
pipedToStdin := (info.Mode() & os.ModeCharDevice) == 0
// takes input from stdin if it exists, otherwise takes input from args (the last argument)
if hasStdin {
if pipedToStdin {
//fmt.Printf("piped: %v\n", args)
if message, err = readStdin(); err != nil {
return
}
} else if len(args) > 0 {
//fmt.Printf("no piped: %v\n", args)
message = args[len(args)-1]
} else {
//fmt.Printf("no data: %v\n", args)
message = ""
}
ret.Message = message
@ -90,20 +94,21 @@ func Init() (ret *Flags, err error) {
}
// readStdin reads from stdin and returns the input as a string or an error
func readStdin() (string, error) {
func readStdin() (ret string, err error) {
reader := bufio.NewReader(os.Stdin)
var input string
var sb strings.Builder
for {
line, err := reader.ReadString('\n')
if err != nil {
if errors.Is(err, io.EOF) {
sb.WriteString(line)
break
}
return "", fmt.Errorf("error reading from stdin: %w", err)
return "", fmt.Errorf("error reading piped message from stdin: %w", err)
}
input += line
sb.WriteString(line)
}
return input, nil
return sb.String(), nil
}
func (o *Flags) BuildChatOptions() (ret *common.ChatOptions) {

11
docker-compose.yml Normal file
View file

@ -0,0 +1,11 @@
version: '3.8'
services:
fabric-api:
build: .
ports:
- "8080:8080"
volumes:
- ./ENV:/root/.config/fabric/.env:ro
environment:
- GIN_MODE=release

View file

@ -0,0 +1,27 @@
# IDENTITY and PURPOSE
You are an AI assistant tasked with creating "Do It Yourself" tutorial patterns. You will carefully analyze each prompt to identify the specific requirements, materials, ingredients, or any other necessary components for the tutorial. You will then organize these elements into a structured format, ensuring clarity and ease of understanding for the user. Your role is to provide comprehensive instructions that guide the user through each step of the DIY process. You will pay close attention to formatting and presentation, making sure the tutorial is accessible and engaging.
Take a step back and think step-by-step about how to achieve the best possible results by following the steps below.
# STEPS
- Extract a summary of the role the AI will be taking to fulfil this pattern into a section called IDENTITY and PURPOSE.
- Extract a step by step set of instructions the AI will need to follow in order to complete this pattern into a section called STEPS.
- Analyze the prompt to determine what format the output should be in.
- Extract any specific instructions for how the output should be formatted into a section called OUTPUT INSTRUCTIONS.
- Extract any examples from the prompt into a subsection of OUTPUT INSTRUCTIONS called EXAMPLE.
# OUTPUT INSTRUCTIONS
- Only output Markdown.
- Ensure you follow ALL these instructions when creating your output.
# INPUT
INPUT:

View file

@ -0,0 +1,45 @@
# IDENTITY and PURPOSE
You are an expert on writing concise, clear, and illuminating technical user stories for new features in complex software programs
# OUTPUT INSTRUCTIONS
Write the users stories in a fashion recognised by other software stakeholders, including product, development, operations and quality assurance
EXAMPLE USER STORY
Description
As a Highlight developer
I want to migrate email templates over to Mustache
So that future upgrades to the messenger service can be made easier
Acceptance Criteria
- Migrate the existing alerting email templates from the instance specific databases over to the messenger templates blob storage.
- Rename each template to a GUID and store in it's own folder within the blob storage
- Store Subject and Body as separate blobs
- Create an upgrade script to change the value of the Alerting.Email.Template local parameter in all systems to the new template names.
- Change the template retrieval and saving for user editing to contact the blob storage rather than the database
- Remove the database tables and code that handles the SQL based templates
- Highlight sends the template name and the details of the body to the Email queue in Service bus
- this is handled by the generic Email Client (if created already)
- This email type will be added to the list of email types that are sent to the messenger service (switch to be removed once all email templates are completed)
- Include domain details as part of payload sent to the messenger service
Note: ensure that Ops know when this work is being done so they are aware of any changes to existing templates
# OUTPUT INSTRUCTIONS
- Write the user story according to the structure above.
- That means the user story should be written in a simple, bulleted style, not in a grandiose, conversational or academic style.
# OUTPUT FORMAT
- Output a full, user story about the content provided using the instructions above.
- The structure should be: Description, Acceptance criteria
- Write in a simple, plain, and clear style, not in a grandiose, conversational or academic style.
- Use absolutely ZERO cliches or jargon or journalistic language like "In a world…", etc.
- Do not use cliches or jargon.
- Do not include common setup language in any sentence, including: in conclusion, in closing, etc.
- Do not output warnings or notes—just the output requested.

View file

@ -1 +1 @@
"1.4.85"
"1.4.94"

View file

@ -2,14 +2,17 @@ package youtube
import (
"context"
"encoding/csv"
"encoding/json"
"flag"
"fmt"
"log"
"net/url"
"os"
"regexp"
"strconv"
"strings"
"time"
"github.com/anaskhan96/soup"
"github.com/danielmiessler/fabric/plugins"
@ -37,38 +40,56 @@ type YouTube struct {
*plugins.PluginBase
ApiKey *plugins.SetupQuestion
service *youtube.Service
normalizeRegex *regexp.Regexp
service *youtube.Service
}
func (o *YouTube) initService() (err error) {
if o.service == nil {
o.normalizeRegex = regexp.MustCompile(`[^a-zA-Z0-9]+`)
ctx := context.Background()
o.service, err = youtube.NewService(ctx, option.WithAPIKey(o.ApiKey.Value))
}
return
}
func (o *YouTube) GetVideoId(url string) (ret string, err error) {
func (o *YouTube) GetVideoOrPlaylistId(url string) (videoId string, playlistId string, err error) {
if err = o.initService(); err != nil {
return
}
pattern := `(?:https?:\/\/)?(?:www\.)?(?:youtube\.com\/(?:[^\/\n\s]+\/\S+\/|(?:v|e(?:mbed)?)\/|\S*?[?&]v=)|youtu\.be\/)([a-zA-Z0-9_-]{11})`
re := regexp.MustCompile(pattern)
match := re.FindStringSubmatch(url)
if len(match) > 1 {
ret = match[1]
} else {
err = fmt.Errorf("invalid YouTube URL, can't get video ID")
// Video ID pattern
videoPattern := `(?:https?:\/\/)?(?:www\.)?(?:youtube\.com\/(?:[^\/\n\s]+\/\S+\/|(?:v|e(?:mbed)?)\/|(?:s(?:horts)\/)|\S*?[?&]v=)|youtu\.be\/)([a-zA-Z0-9_-]*)`
videoRe := regexp.MustCompile(videoPattern)
videoMatch := videoRe.FindStringSubmatch(url)
if len(videoMatch) > 1 {
videoId = videoMatch[1]
}
// Playlist ID pattern
playlistPattern := `[?&]list=([a-zA-Z0-9_-]+)`
playlistRe := regexp.MustCompile(playlistPattern)
playlistMatch := playlistRe.FindStringSubmatch(url)
if len(playlistMatch) > 1 {
playlistId = playlistMatch[1]
}
if videoId == "" && playlistId == "" {
err = fmt.Errorf("invalid YouTube URL, can't get video or playlist ID: '%s'", url)
}
return
}
func (o *YouTube) GrabTranscriptForUrl(url string, language string) (ret string, err error) {
var videoId string
if videoId, err = o.GetVideoId(url); err != nil {
var playlistId string
if videoId, playlistId, err = o.GetVideoOrPlaylistId(url); err != nil {
return
} else if videoId == "" && playlistId != "" {
err = fmt.Errorf("URL is a playlist, not a video")
return
}
return o.GrabTranscript(videoId, language)
}
@ -172,7 +193,11 @@ func (o *YouTube) GrabDurationForUrl(url string) (ret int, err error) {
}
var videoId string
if videoId, err = o.GetVideoId(url); err != nil {
var playlistId string
if videoId, playlistId, err = o.GetVideoOrPlaylistId(url); err != nil {
return
} else if videoId == "" && playlistId != "" {
err = fmt.Errorf("URL is a playlist, not a video")
return
}
return o.GrabDuration(videoId)
@ -203,7 +228,11 @@ func (o *YouTube) GrabDuration(videoId string) (ret int, err error) {
func (o *YouTube) Grab(url string, options *Options) (ret *VideoInfo, err error) {
var videoId string
if videoId, err = o.GetVideoId(url); err != nil {
var playlistId string
if videoId, playlistId, err = o.GetVideoOrPlaylistId(url); err != nil {
return
} else if videoId == "" && playlistId != "" {
err = fmt.Errorf("URL is a playlist, not a video")
return
}
@ -232,6 +261,109 @@ func (o *YouTube) Grab(url string, options *Options) (ret *VideoInfo, err error)
return
}
// FetchPlaylistVideos fetches all videos from a YouTube playlist.
func (o *YouTube) FetchPlaylistVideos(playlistID string) (ret []*VideoMeta, err error) {
if err = o.initService(); err != nil {
return
}
nextPageToken := ""
for {
call := o.service.PlaylistItems.List([]string{"snippet"}).PlaylistId(playlistID).MaxResults(50)
if nextPageToken != "" {
call = call.PageToken(nextPageToken)
}
var response *youtube.PlaylistItemListResponse
if response, err = call.Do(); err != nil {
return
}
for _, item := range response.Items {
videoID := item.Snippet.ResourceId.VideoId
title := item.Snippet.Title
ret = append(ret, &VideoMeta{videoID, title, o.normalizeFileName(title)})
}
nextPageToken = response.NextPageToken
if nextPageToken == "" {
break
}
time.Sleep(1 * time.Second) // Pause to respect API rate limit
}
return
}
// SaveVideosToCSV saves the list of videos to a CSV file.
func (o *YouTube) SaveVideosToCSV(filename string, videos []*VideoMeta) (err error) {
var file *os.File
if file, err = os.Create(filename); err != nil {
return
}
defer file.Close()
writer := csv.NewWriter(file)
defer writer.Flush()
// Write headers
if err = writer.Write([]string{"VideoID", "Title"}); err != nil {
return
}
// Write video data
for _, record := range videos {
if err = writer.Write([]string{record.Id, record.Title}); err != nil {
return
}
}
return
}
// FetchAndSavePlaylist fetches all videos in a playlist and saves them to a CSV file.
func (o *YouTube) FetchAndSavePlaylist(playlistID, filename string) (err error) {
var videos []*VideoMeta
if videos, err = o.FetchPlaylistVideos(playlistID); err != nil {
err = fmt.Errorf("error fetching playlist videos: %v", err)
return
}
if err = o.SaveVideosToCSV(filename, videos); err != nil {
err = fmt.Errorf("error saving videos to CSV: %v", err)
return
}
fmt.Println("Playlist saved to", filename)
return
}
func (o *YouTube) FetchAndPrintPlaylist(playlistID string) (err error) {
var videos []*VideoMeta
if videos, err = o.FetchPlaylistVideos(playlistID); err != nil {
err = fmt.Errorf("error fetching playlist videos: %v", err)
return
}
fmt.Printf("Playlist: %s\n", playlistID)
fmt.Printf("VideoId: Title\n")
for _, video := range videos {
fmt.Printf("%s: %s\n", video.Id, video.Title)
}
return
}
func (o *YouTube) normalizeFileName(name string) string {
return o.normalizeRegex.ReplaceAllString(name, "_")
}
type VideoMeta struct {
Id string
Title string
TitleNormalized string
}
type Options struct {
Duration bool
Transcript bool

View file

@ -1,3 +1,3 @@
package main
var version = "v1.4.87"
var version = "v1.4.94"