# -*- mode: sh; sh-shell: bash -*-
# vim: set ft=bash:
# shellcheck shell=bash

### A persistent database which memoizes command results
###
### It records the stdout, stderr and state/exitcode commands and functions.  Commands and
### functions that are used with the memodb must be pure as their result must only depend on
### the arguments passed and the directory they are called in. The content of the directory is
### hashed for the database. This requires properly set up '.gitignore' rules to ignore any
### build artifacts. Otherwise subsequent instances will not use the same memodb and re-run
### commands.
###
### This is used for
###  1. Persist/cache command results throughout multiple invocations
###  2. Implement background processing where one invocation can schedule commands to be
###     executed in the background and a later invocation can collect the results

# Internals
#
# the memo dir .bar.db/<timestamp>-<treehash>/ contains:
#  - .bar.lock - the lock per tag/operation the memo dir
#  - .bar.log  - log for the cached memos holding "state cmdhash pwd//#cmd args.."
#                Where state is either a numeric exit code alphabetic for backgrounding.
#  - <cmdhash>.stdout   - stdout of commands
#  - <cmdhash>.stderr   - stderr of commands
#
# At runtime the MEMODB stores a "pwd//#cmd args" : "state hash" relation.  Note that '//#' is
# a fancy delimiter here as it should not appear in paths or in front of commands When a log
# exists then at startup then the MEMODB is replayed from the log.

# PLANNED: for now only one background process is spawned eventually there will be one per tag

# We may have multiple memo logs that are distinguished by their name and the tree hash.
# Each memo log is used to cache the result of tests and to schedule background jobs.
# A memo log is appended to after each test is run.

require lock git_lib

declare -gx MEMODB_KEEP="${MEMODB_KEEP:-5}"                      ## How many trees to keep.
declare -gx MEMODB_BACKGROUNDING="${MEMODB_BACKGROUNDING:-true}" ## Global backgrounding flag.
declare -gA MEMODB_INITIALIZED_TAGS=()

function memodb_init # [tag] - initializes a memo log
{
    # (re-) initializes a memo log, acquires a lock
    trace "$*"

    local tag="${1:-bg}"

    # initialize only once
    [[ -v MEMODB_INITIALIZED_TAGS[$tag] ]] && return 0
    MEMODB_INITIALIZED_TAGS[$tag]=true

    # main lock will be released hand-over-hand
    lock_wait "$BAR_TOPLEVEL/.bar"
    mkdir -p "$BAR_TOPLEVEL/.bar.db"

    # PLANNED: make tree_hash a rule that can be overridden by the user
    # shellcheck disable=SC2155
    local tree_hash="$(git_tree_hash -c --other)"

    # shellcheck disable=SC2155
    declare -gx MEMODB_DIR="$(find "$BAR_TOPLEVEL/.bar.db" -type d -name "*-$tree_hash" | tail -1)"
    if [[ -z "$MEMODB_DIR" ]]; then
        MEMODB_DIR="$BAR_TOPLEVEL/.bar.db/$BAR_TIMESTAMP-$tree_hash"
        info "creating: $MEMODB_DIR"
        mkdir -p "$MEMODB_DIR"
    else
        info "using existing: $MEMODB_DIR"
    fi
    lock_next "$BAR_TOPLEVEL/.bar" "$MEMODB_DIR/.memodb"

    readonly MEMODB_DIR

    # MEMODB_LOG keeps "$exitcode $cmdhash $PWD//#$*"
    declare -gx MEMODB_LOG="$MEMODB_DIR/.log"
    declare -gAx MEMODB      # the actual state: "pwd//#cmd args":"state cmdhash"

    if [[ -f "$MEMODB_LOG" ]]; then
        # replay the log
        local state
        local cmdhash
        local rest
        while read -r state cmdhash rest; do
            MEMODB[$rest]="$state $cmdhash"
        done <"$MEMODB_LOG"
    else
        # new memo log
        touch "$MEMODB_LOG"
    fi

    rule POSTPROCESS: -- memodb_start_background_processing
    rule CLEANUP: -- "cd '$BAR_TOPLEVEL'; lock_remove '$MEMODB_DIR/.memodb'; memodb_gc;"
}

function memodb_cached # cmd args.. - returns the last cached log entry memo
{
    local entry="${MEMODB[$PWD//#$*]:-}"
    echo "${entry%% *}"
}

function memodb_cmdhash # cmd args.. - returns the hash of $PWD, a command and its args
{
    local entry="${MEMODB[${PWD}//#$*]:-}"

    if [[ -n "$entry" ]]; then
        entry="${entry#* }"
        echo "${entry%% *}"
    else
        hash_args "$*"
    fi
}

function memodb_replay_stdio # <cmdhash> - replays the stdout and stderr of a command
{
    trace "$*"
    cat "$MEMODB_DIR/$1.stdout"
    cat "$MEMODB_DIR/$1.stderr" 1>&2
}

function memodb_log # state hash [pwd] cmd args.. - logs the state of a command
{
    local state="$1"
    local hash="$2"
    shift 2
    
    # If third argument looks like a path (starts with /), use it as pwd
    local log_pwd="$PWD"
    if [[ "$1" =~ ^/ ]]; then
        log_pwd="$1"
        shift
    fi

    trace "$state $* >> $MEMODB_LOG"
    echo "$state $hash $log_pwd//#$*" >>"$MEMODB_LOG"
    MEMODB[$log_pwd//#$*]="$state $hash"
}

# PLANNED: function memodb_eval # [--tag <tag>]
function memodb_eval ## <cmd> [args..] - Execute a command in foreground and stores its results.
{
    ## The command is only executed on the first time the 'memodb_eval' is called with the
    ## same options in the same directory. Any subsequent call will replay the stored results.
    memodb_init ''

    # shellcheck disable=SC2155
    local cmdhash="$(memodb_cmdhash "$*")"

    # shellcheck disable=SC2155
    local cached_state="$(memodb_cached "$*")"
    case "$cached_state" in
    [[:alpha:]]*)
        # already scheduled in the background, just return
        debug "bg scheduled $*"
        return 0
        ;;
    0)
        # already run successfully, just return
        debug "cached success: $*"
        memodb_replay_stdio "$cmdhash"
        return 0
        ;;
    ?*)
        # already run with failure
        debug "cached failed: $*"
        memodb_replay_stdio "$cmdhash"
        return "$cached_state"
        ;;
    esac

    debug "eval: $*"
    local rc=0
    rule_eval "$@"  1> >(tee "$MEMODB_DIR/$cmdhash.stdout") 2> >(tee "$MEMODB_DIR/$cmdhash.stderr" 1>&2 ) || rc=$?
    memodb_log "$rc" "$cmdhash" "$*"
    return $rc
}

# list of schedules commands as "$cmdhash $PWD//#$*"
declare -gax MEMODB_SCHEDULED=()

# TODO: --tag <tag>
function memodb_schedule ## <cmd> [args..] - Schedules a command to be run in background.
{
    ## Will 'memodb_eval' the command when 'MEMODB_BACKGROUNDING' is not true.
    trace "$*"
    memodb_init ''
    if [[ "$MEMODB_BACKGROUNDING" = true ]]; then
        # shellcheck disable=SC2155
        local cmdhash="$(memodb_cmdhash "$*")"

        # shellcheck disable=SC2155
        local cached_state="$(memodb_cached "$*")"
        case "$cached_state" in
        [[:alpha:]]*)
            # already scheduled in the background, just return
            debug "bg scheduled $*"
            return 0
            ;;
        0)
            # already cached successfully, just return
            debug "cached success: $*"
            memodb_replay_stdio "$cmdhash"
            return 0
            ;;
        ?*)
            # already cached with failure
            debug "cached failed: $*"
            memodb_replay_stdio "$cmdhash"
            return "$cached_state"
            ;;
        esac

        memodb_log "bg" "$cmdhash" "$@"
        # TODO: should become "$tag $cmdhash $PWD//#$*"
        MEMODB_SCHEDULED+=("$cmdhash $PWD//#$*")
    else
        memodb_eval "$@"
    fi
}

function memodb_analyze # cmd args.. - analyzes tree hash mismatches
{
    error ""
    error "Tree hash mismatch - memodb result not available."
    error "Searching for: $PWD//#$*"
    error ""
    
    # Show what's actually in the memodb
    if [[ -f "$MEMODB_LOG" ]]; then
        error "Available results in memodb:"
        while IFS= read -r line; do
            local state="${line%% *}"
            local rest="${line#* }"
            rest="${rest#* }"
            if [[ "$state" =~ ^[0-9]+$ ]]; then
                error "  $rest (exit: $state)"
            fi
        done < "$MEMODB_LOG"
        error ""
    fi
    
    error "This usually means:"
    error "  1. The command was run in a different directory, or"
    error "  2. Untracked files are changing the tree hash"
    error ""
    
    # Show untracked non-ignored files (these affect the tree hash)
    local untracked_files
    untracked_files="$(git_ls_files --others || true)"
    
    if [[ -n "$untracked_files" ]]; then
        error "Untracked files found (these affect the tree hash):"
        echo "$untracked_files" | while IFS= read -r file; do
            error "  $file"
        done
        error ""
        error "Consider adding these patterns to .gitignore:"
        # Extract unique directory patterns and file patterns
        echo "$untracked_files" | while IFS= read -r file; do
            # Show both the specific file and directory patterns
            if [[ "$file" =~ / ]]; then
                # Has a directory component
                local dir="${file%/*}"
                echo "$dir/"
            fi
            # Also show file extension patterns
            if [[ "$file" =~ \. ]]; then
                local ext="${file##*.}"
                echo "*.$ext"
            fi
        done | sort -u | while IFS= read -r pattern; do
            error "  $pattern"
        done
    else
        error "No untracked files found."
        error "The command may have been run from a different directory."
    fi
}

function memodb_replay_bg_log
{
    [[ "$MEMODB_BACKGROUNDING" = true && -f "$MEMODB_DIR/.bg.log" ]] || return 0

    local state
    local cmdhash
    local rest
    while read -r state cmdhash rest; do
        trace "MEMODB merge: $state $rest >> $MEMODB_LOG"
        MEMODB[$rest]="$state $cmdhash"
        echo "$state $cmdhash $rest" >>"$MEMODB_LOG"
    done <"$MEMODB_DIR/.bg.log"
    rm "$MEMODB_DIR/.bg.log"
}

# TODO: --tag <tag>
function memodb_result ## <cmd> [args..] - Retrieves the background results.
{
    ## Waits for the backgrounding to be completed. Merges the results into the current memodb.
    ## Replays the result (stdout/stderr/exitcode) of the given 'cmd [args..]'.
    ## 'cmd [args..]' must be the same as used when scheduling it to the background.
    ## Will error when the command was not previously scheduled and backgrounding is enabled.
    trace "$*"
    memodb_init ''
    # shellcheck disable=SC2155
    local cmdhash="$(memodb_cmdhash "$*")"

    memodb_replay_bg_log

    # shellcheck disable=SC2155
    local cached_state="$(memodb_cached "$*")"
    case "$cached_state" in
    [[:alpha:]]*)
        # fall through
        ;;
    0)
        # already run successfully, just return
        debug "cached success: $*"
        memodb_replay_stdio "$cmdhash"
        return 0
        ;;
    ?*)
        # already run with failure
        debug "cached failed: $*"
        memodb_replay_stdio "$cmdhash"
        return "$cached_state"
        ;;
    esac

    memodb_analyze "$@"
    die "No result available for '$*'"
}

function memodb_start_background_processing
{
    [[ "${#MEMODB_SCHEDULED[@]}" = 0 ]] && return 0
    trace "$*"

    # TODO: one process per tag
    {
        lock_receive "$MEMODB_DIR/.memodb"

        MEMODB_LOG="$MEMODB_DIR/.bg.log"
        touch "$MEMODB_LOG"

        for job in "${MEMODB_SCHEDULED[@]}"; do
            local cmdhash="${job%% *}"
            job="${job#* }"
            local directory="${job%//#*}"
            local cmd="${job#*//#}"

            cd "$directory" || die "can't change to $directory"
            # shellcheck disable=SC2155
            local cached_state="$(memodb_cached "$cmd")"
            if [[ "$cached_state" = [[:alpha:]]* ]]; then
                debug "bgeval: $cmd"
                # shellcheck disable=2086
                if rule_eval $cmd 1>"$MEMODB_DIR/$cmdhash.stdout" 2>"$MEMODB_DIR/$cmdhash.stderr" ; then
                    debug "success: $cmd"
                    memodb_log 0 "$cmdhash" "$directory" $cmd
                else
                    local rc=$?
                    debug "fail: $cmd"
                    memodb_log "$rc" "$cmdhash" "$directory" $cmd
                fi
            fi
        done

        lock_remove "$MEMODB_DIR/.memodb"
    } 1>"$MEMODB_DIR/.bg.stdout" 2>"$MEMODB_DIR/.bg.stderr" &
    lock_send "$MEMODB_DIR/.memodb" $!
    disown $!
    note "Background processing started!"
}

function memodb_clean # - cleanup/delete the memodb state
{
    local memodb
    lock_wait "$BAR_TOPLEVEL/.bar"
    if [[ -d "$BAR_TOPLEVEL/.bar.db" ]]; then
        find "$BAR_TOPLEVEL/.bar.db" -name "*-*" -type d -print0 |
            while IFS= read -r -d '' memodb; do
                if lock_try_norec "$memodb/.memodb"; then
                    rm -rf "$memodb"
                fi
            done

        rm -d "$BAR_TOPLEVEL/.bar.db" 2>/dev/null || true
    fi
    lock_remove "$BAR_TOPLEVEL/.bar"
}

function memodb_gc # - cleanup old memodbs
{
    local memodb
    lock_wait "$BAR_TOPLEVEL/.bar"
    find "$BAR_TOPLEVEL/.bar.db" -name "*-*" -type d -print0 | sort -z -n | head -z -n -$((MEMODB_KEEP)) |
        while IFS= read -r -d '' memodb; do
            if lock_try_norec "$memodb/.memodb"; then
                rm -rf "$memodb"
            fi
        done
    lock_remove "$BAR_TOPLEVEL/.bar"
}

# function memo_exit ## - exits a memodb, remove the lock, but keeps the log
# {
#     if [[ -n "$MEMO_LOCK" ]]; then
#         unset MEMO_LOG
#         unset MEMO_DB
#         lock_remove "$MEMO_LOCK"
#     fi
# }
#
# PLANNED: backgrounding git_hook_matches "pre-commit" "pre-merge-commit" && background_schedule && return 0
#                        git_hook_matches "commit-msg" && background_wait && return $(background_result)
# PLANNED: reintroduce tags for backgrounding, create one background process per tag
