feat: add database synchronization and improve changelog processing workflow

## CHANGES

- Add database sync command with comprehensive validation
- Implement version and commit existence checking methods
- Enhance time parsing with RFC3339Nano fallback support
- Cache fetched PRs during changelog entry creation
- Remove individual incoming files using git operations
- Add sync-db flag for database integrity validation
- Improve commit-PR mapping verification process
- Exclude incoming directory from workflow trigger paths

.github/workflows/update-version-and-create-tag.yml

@@ -9,6 +9,7 @@ on:
       - "**/*.md"
       - "data/strategies/**"
       - "cmd/generate_changelog/*.db"
+      - "cmd/generate_changelog/incoming/*.txt"
       - "scripts/pattern_descriptions/*.json"
       - "web/static/data/pattern_descriptions.json"
 

cmd/generate_changelog/incoming/1640.txt

@@ -1,7 +0,0 @@
-### PR [#1640](https://github.com/danielmiessler/Fabric/pull/1640) by [ksylvan](https://github.com/ksylvan): Implement Automated Changelog System for CI/CD Integration
-
-- Add automated changelog processing for CI/CD integration with comprehensive test coverage and GitHub client validation methods
-- Implement release aggregation for incoming files with git operations for staging changes and support for version detection from nix files
-- Change push behavior from opt-out to opt-in with GitHub token authentication and automatic repository detection
-- Enhance changelog generation to avoid duplicate commit entries by extracting PR numbers and filtering commits already included via PR files
-- Add version parameter requirement for PR processing with commit SHA tracking to prevent duplicate entries and improve formatting consistency

cmd/generate_changelog/incoming/1641.txt

@@ -1,5 +0,0 @@
-### PR [#1641](https://github.com/danielmiessler/Fabric/pull/1641) by [ksylvan](https://github.com/ksylvan): Fix Fabric Web timeout error
-
-- Chore: extend proxy timeout in `vite.config.ts` to 15 minutes
-- Increase `/api` proxy timeout to 900,000 ms
-- Increase `/names` proxy timeout to 900,000 ms

cmd/generate_changelog/internal/cache/cache.go

@@ -201,7 +201,11 @@ func (c *Cache) GetVersions() (map[string]*git.Version, error) {
 		}
 
 		if dateStr.Valid {
-			v.Date, _ = time.Parse(time.RFC3339, dateStr.String)
+			// Try RFC3339Nano first (for nanosecond precision), then fall back to RFC3339
+			v.Date, err = time.Parse(time.RFC3339Nano, dateStr.String)
+			if err != nil {
+				v.Date, _ = time.Parse(time.RFC3339, dateStr.String)
+			}
 		}
 
 		if prNumbersJSON != "" {
@@ -260,6 +264,26 @@ func (c *Cache) Clear() error {
 	return nil
 }
 
+// VersionExists checks if a version already exists in the cache
+func (c *Cache) VersionExists(version string) (bool, error) {
+	var count int
+	err := c.db.QueryRow("SELECT COUNT(*) FROM versions WHERE name = ?", version).Scan(&count)
+	if err != nil {
+		return false, err
+	}
+	return count > 0, nil
+}
+
+// CommitExists checks if a commit already exists in the cache
+func (c *Cache) CommitExists(hash string) (bool, error) {
+	var count int
+	err := c.db.QueryRow("SELECT COUNT(*) FROM commits WHERE sha = ?", hash).Scan(&count)
+	if err != nil {
+		return false, err
+	}
+	return count > 0, nil
+}
+
 // GetLastPRSync returns the timestamp of the last PR sync
 func (c *Cache) GetLastPRSync() (time.Time, error) {
 	var timestamp string

cmd/generate_changelog/internal/changelog/generator.go

@@ -697,3 +697,110 @@ func hashContent(content string) string {
 	hash := sha256.Sum256([]byte(content))
 	return fmt.Sprintf("%x", hash)
 }
+
+// SyncDatabase performs a comprehensive database synchronization and validation
+func (g *Generator) SyncDatabase() error {
+	if g.cache == nil {
+		return fmt.Errorf("cache is disabled, cannot sync database")
+	}
+
+	fmt.Fprintf(os.Stderr, "🔄 Starting database synchronization...\n")
+
+	// Step 1: Force PR sync (reuse existing logic)
+	fmt.Fprintf(os.Stderr, "📥 Forcing PR sync from GitHub...\n")
+	g.cfg.ForcePRSync = true
+	if err := g.fetchPRs(); err != nil {
+		return fmt.Errorf("failed to sync PRs: %w", err)
+	}
+
+	// Step 2: Rebuild git history and verify versions/commits completeness
+	fmt.Fprintf(os.Stderr, "🔍 Verifying git history and version completeness...\n")
+	if err := g.syncGitHistory(); err != nil {
+		return fmt.Errorf("failed to sync git history: %w", err)
+	}
+
+	// Step 3: Verify commit-PR mappings
+	fmt.Fprintf(os.Stderr, "🔗 Verifying commit-PR mappings...\n")
+	if err := g.verifyCommitPRMappings(); err != nil {
+		return fmt.Errorf("failed to verify commit-PR mappings: %w", err)
+	}
+
+	fmt.Fprintf(os.Stderr, "✅ Database synchronization completed successfully!\n")
+	return nil
+}
+
+// syncGitHistory walks the complete git history and ensures all versions and commits are cached
+func (g *Generator) syncGitHistory() error {
+	// Walk complete git history (reuse existing logic)
+	versions, err := g.gitWalker.WalkHistory()
+	if err != nil {
+		return fmt.Errorf("failed to walk git history: %w", err)
+	}
+
+	// Save only new versions and commits (preserve existing data)
+	var newVersions, newCommits int
+	for _, version := range versions {
+		// Only save version if it doesn't exist
+		exists, err := g.cache.VersionExists(version.Name)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Warning: Failed to check version %s existence: %v\n", version.Name, err)
+			continue
+		}
+		if !exists {
+			if err := g.cache.SaveVersion(version); err != nil {
+				fmt.Fprintf(os.Stderr, "Warning: Failed to save version %s: %v\n", version.Name, err)
+			} else {
+				newVersions++
+			}
+		}
+
+		// Only save commits that don't exist
+		for _, commit := range version.Commits {
+			exists, err := g.cache.CommitExists(commit.SHA)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "Warning: Failed to check commit %s existence: %v\n", commit.SHA, err)
+				continue
+			}
+			if !exists {
+				if err := g.cache.SaveCommit(commit, version.Name); err != nil {
+					fmt.Fprintf(os.Stderr, "Warning: Failed to save commit %s: %v\n", commit.SHA, err)
+				} else {
+					newCommits++
+				}
+			}
+		}
+	}
+
+	// Update last processed tag
+	if latestTag, err := g.gitWalker.GetLatestTag(); err == nil && latestTag != "" {
+		if err := g.cache.SetLastProcessedTag(latestTag); err != nil {
+			fmt.Fprintf(os.Stderr, "Warning: Failed to update last processed tag: %v\n", err)
+		}
+	}
+
+	fmt.Fprintf(os.Stderr, "   Added %d new versions and %d new commits (preserved existing data)\n", newVersions, newCommits)
+	return nil
+}
+
+// verifyCommitPRMappings ensures all PR commits have proper mappings
+func (g *Generator) verifyCommitPRMappings() error {
+	// Get all cached PRs
+	allPRs, err := g.cache.GetAllPRs()
+	if err != nil {
+		return fmt.Errorf("failed to get cached PRs: %w", err)
+	}
+
+	// Convert to slice for batch operations (reuse existing logic)
+	var prSlice []*github.PR
+	for _, pr := range allPRs {
+		prSlice = append(prSlice, pr)
+	}
+
+	// Save commit-PR mappings (reuse existing logic)
+	if err := g.cache.SaveCommitPRMappings(prSlice); err != nil {
+		return fmt.Errorf("failed to save commit-PR mappings: %w", err)
+	}
+
+	fmt.Fprintf(os.Stderr, "   Verified mappings for %d PRs\n", len(prSlice))
+	return nil
+}

cmd/generate_changelog/internal/changelog/processing.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/danielmiessler/fabric/cmd/generate_changelog/internal/git"
+	"github.com/danielmiessler/fabric/cmd/generate_changelog/internal/github"
 )
 
 // ProcessIncomingPR processes a single PR for changelog entry creation
@@ -89,6 +90,8 @@ func (g *Generator) CreateNewChangelogEntry(version string) error {
 	// Extract PR numbers and their commit SHAs from processed files to avoid including their commits as "direct"
 	processedPRs := make(map[int]bool)
 	processedCommitSHAs := make(map[string]bool)
+	var fetchedPRs []*github.PR
+	var prNumbers []int
 
 	for _, file := range files {
 		// Extract PR number from filename (e.g., "1640.txt" -> 1640)
@@ -96,9 +99,11 @@ func (g *Generator) CreateNewChangelogEntry(version string) error {
 		if prNumStr := strings.TrimSuffix(filename, ".txt"); prNumStr != filename {
 			if prNum, err := strconv.Atoi(prNumStr); err == nil {
 				processedPRs[prNum] = true
+				prNumbers = append(prNumbers, prNum)
 
 				// Fetch the PR to get its commit SHAs
 				if pr, err := g.ghClient.GetPRWithCommits(prNum); err == nil {
+					fetchedPRs = append(fetchedPRs, pr)
 					for _, commit := range pr.Commits {
 						processedCommitSHAs[commit.SHA] = true
 					}
@@ -132,12 +137,44 @@ func (g *Generator) CreateNewChangelogEntry(version string) error {
 	}
 
 	if g.cache != nil {
+		// Cache the fetched PRs using the same logic as normal changelog generation
+		if len(fetchedPRs) > 0 {
+			// Save PRs to cache
+			if err := g.cache.SavePRBatch(fetchedPRs); err != nil {
+				fmt.Fprintf(os.Stderr, "Warning: Failed to cache PRs: %v\n", err)
+			}
+
+			// Save SHA→PR mappings for lightning-fast git operations
+			if err := g.cache.SaveCommitPRMappings(fetchedPRs); err != nil {
+				fmt.Fprintf(os.Stderr, "Warning: Failed to cache commit mappings: %v\n", err)
+			}
+
+			// Save individual commits to cache for each PR
+			for _, pr := range fetchedPRs {
+				for _, commit := range pr.Commits {
+					// Convert github.PRCommit to git.Commit
+					gitCommit := &git.Commit{
+						SHA:      commit.SHA,
+						Message:  commit.Message,
+						Author:   commit.Author,
+						Email:    "",         // Not available from GitHub API
+						Date:     time.Now(), // Use current time as fallback
+						IsMerge:  false,      // We don't have this info from GitHub API
+						PRNumber: pr.Number,
+					}
+					if err := g.cache.SaveCommit(gitCommit, version); err != nil {
+						fmt.Fprintf(os.Stderr, "Warning: Failed to save commit %s to cache: %v\n", commit.SHA, err)
+					}
+				}
+			}
+		}
+
 		// Create a proper new version entry for the database
 		newVersionEntry := &git.Version{
 			Name:      version,
 			Date:      time.Now(),
-			CommitSHA: "",      // Will be set when the release commit is made
-			PRNumbers: []int{}, // Could be extracted from incoming files if needed
+			CommitSHA: "",        // Will be set when the release commit is made
+			PRNumbers: prNumbers, // Now we have the actual PR numbers
 			AISummary: content.String(),
 		}
 
@@ -147,8 +184,19 @@ func (g *Generator) CreateNewChangelogEntry(version string) error {
 	}
 
 	for _, file := range files {
-		if err := os.Remove(file); err != nil {
-			fmt.Fprintf(os.Stderr, "Warning: Failed to remove %s: %v\n", file, err)
+		// Convert to relative path for git operations
+		relativeFile, err := filepath.Rel(g.cfg.RepoPath, file)
+		if err != nil {
+			relativeFile = file
+		}
+
+		// Use git remove to handle both filesystem and git index
+		if err := g.gitWalker.RemoveFile(relativeFile); err != nil {
+			fmt.Fprintf(os.Stderr, "Warning: Failed to remove %s from git: %v\n", relativeFile, err)
+			// Fallback to filesystem-only removal
+			if err := os.Remove(file); err != nil {
+				fmt.Fprintf(os.Stderr, "Warning: Failed to remove %s from filesystem: %v\n", file, err)
+			}
 		}
 	}
 
@@ -391,17 +439,8 @@ func (g *Generator) stageChangesForRelease() error {
 		return fmt.Errorf("failed to add %s: %w", relativeCacheFile, err)
 	}
 
-	// Remove incoming directory if it exists
-	if g.cfg.IncomingDir != "" {
-		relativeIncomingDir, err := filepath.Rel(g.cfg.RepoPath, g.cfg.IncomingDir)
-		if err != nil {
-			relativeIncomingDir = g.cfg.IncomingDir
-		}
-
-		if err := g.gitWalker.RemoveFile(relativeIncomingDir); err != nil {
-			return fmt.Errorf("failed to remove incoming directory %s: %w", relativeIncomingDir, err)
-		}
-	}
+	// Note: Individual incoming files are now removed during the main processing loop
+	// No need to remove the entire directory here
 
 	return nil
 }

cmd/generate_changelog/internal/config/config.go

@@ -16,4 +16,5 @@ type Config struct {
 	ProcessPRsVersion string
 	IncomingDir       string
 	Push              bool
+	SyncDB            bool
 }

cmd/generate_changelog/main.go

@@ -41,6 +41,7 @@ func init() {
 	rootCmd.Flags().StringVar(&cfg.ProcessPRsVersion, "process-prs", "", "Process all incoming PR files for release (provide version like v1.4.262)")
 	rootCmd.Flags().StringVar(&cfg.IncomingDir, "incoming-dir", "./cmd/generate_changelog/incoming", "Directory for incoming PR files")
 	rootCmd.Flags().BoolVar(&cfg.Push, "push", false, "Enable automatic git push after creating an incoming entry")
+	rootCmd.Flags().BoolVar(&cfg.SyncDB, "sync-db", false, "Synchronize and validate database integrity with git history and GitHub PRs")
 }
 
 func run(cmd *cobra.Command, args []string) error {
@@ -65,6 +66,10 @@ func run(cmd *cobra.Command, args []string) error {
 		return generator.CreateNewChangelogEntry(cfg.ProcessPRsVersion)
 	}
 
+	if cfg.SyncDB {
+		return generator.SyncDatabase()
+	}
+
 	output, err := generator.Generate()
 	if err != nil {
 		return fmt.Errorf("failed to generate changelog: %w", err)