⬆️ Updated dependencies

- Moved mappings into Sb1 impl
- Moved Actual types to @common
- Moved createDirIfMissing to respective functions
- Refactored main into multiple functions
- Moved create db into Sb1impl and close
- Log requests on info

.gitea/workflows/deploy.yml

@@ -25,6 +25,8 @@ jobs:
       LOG_LEVEL: ${{ vars.LOG_LEVEL }}
       DB_DIRECTORY: ${{ vars.DB_DIRECTORY }}
       DB_FILENAME: ${{ vars.DB_FILENAME }}
+      TRANSACTION_RELATIVE_FROM_DATE: ${{ vars.TRANSACTION_RELATIVE_FROM_DATE || 'N/A' }}
+      TRANSACTION_RELATIVE_TO_DATE: ${{ vars.TRANSACTION_RELATIVE_TO_DATE || 'N/A' }}
 
     steps:
       - name: Check out repository code

docker-compose.yml

@@ -1,7 +1,7 @@
 services:
   server:
     container_name: actual_sparebank1_cronjob
-    restart: no
+    restart: unless-stopped
     build:
       context: .
     environment:
@@ -16,13 +16,15 @@ services:
       - BANK_OAUTH_CLIENT_SECRET
       - BANK_ACCOUNT_IDS
       - LOG_LEVEL
-      - DB_DIRECTORY # Required for Docker Compose
+      - DB_DIRECTORY
       - DB_FILENAME
       - TRANSACTION_RELATIVE_FROM_DATE
       - TRANSACTION_RELATIVE_TO_DATE
     volumes:
-      # TODO CACHE directory in volume?
+      - cache:/app/${ACTUAL_DATA_DIR:-.cache}
-      - data:/${DB_DIRECTORY}
+      - data:/app/${DB_DIRECTORY:-data}
 
+# TODO change volume name from hostexecutor-*
 volumes:
+  cache:
   data:

package.json

@@ -14,24 +14,24 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "@actual-app/api": "^25.2.1",
+    "@actual-app/api": "^25.5.0",
-    "@dotenvx/dotenvx": "^1.35.0",
+    "@dotenvx/dotenvx": "^1.44.0",
-    "better-sqlite3": "^11.8.1",
+    "better-sqlite3": "^11.10.0",
-    "cron": "^3.5.0",
+    "cron": "^4.3.0",
     "dayjs": "^1.11.13",
-    "dotenv": "^16.4.7",
+    "dotenv": "^16.5.0",
-    "prettier": "^3.5.0",
+    "prettier": "^3.5.3",
-    "tsx": "^4.19.2"
+    "tsx": "^4.19.4"
   },
   "devDependencies": {
     "@jest/globals": "^29.7.0",
-    "@types/better-sqlite3": "^7.6.12",
+    "@types/better-sqlite3": "^7.6.13",
     "@types/jest": "^29.5.14",
-    "@types/node": "^22.13.1",
+    "@types/node": "^22.15.18",
     "jest": "^29.7.0",
-    "ts-jest": "^29.2.5",
+    "ts-jest": "^29.3.3",
     "ts-node": "^10.9.2",
-    "typescript": "^5.7.3"
+    "typescript": "^5.8.3"
   },
   "prettier": {
     "semi": false,

packages/common/types.ts

@@ -0,0 +1,66 @@
+import type { Dayjs } from "dayjs"
+import type { UUID } from "node:crypto"
+import type { TransactionEntity } from "@actual-app/api/@types/loot-core/types/models"
+
+/**
+ * Defines how to interact with the bank
+ */
+export interface Bank {
+  /**
+   * Fetch all transactions in the specified days, from the given accounts
+   * @param interval Which days to fetch transactions for
+   * @param accountKeys The id of the accounts to fetch transactions from
+   * @returns An array of all transactions
+   */
+  fetchTransactions: (
+    interval: Interval,
+    ...accountKeys: ReadonlyArray<string>
+  ) => Promise<ReadonlyArray<ActualTransaction>>
+
+  /**
+   * Shutdown resources
+   */
+  shutdown: () => Promise<void> | void
+}
+
+export interface Interval {
+  fromDate: Dayjs
+  toDate: Dayjs
+}
+
+/**
+ * Describes how to interact with ActualBudget
+ */
+export interface Actual {
+  /**
+   * Import transactions following the rules defined in the ActualBudget instance
+   * If the transactions exists, it will be updated, or no change should be done.
+   * @param accountId The ActualBudget id to upload to
+   * @param transactions The transactions to import
+   * @returns An object describing what changed
+   */
+  importTransactions: (
+    accountId: UUID,
+    transactions: Iterable<ActualTransaction>,
+  ) => Promise<ImportTransactionsResponse>
+
+  /**
+   * Disconnect from ActualBudget and release resources
+   */
+  shutdown: () => Promise<void>
+}
+
+export interface ActualTransaction extends TransactionEntity {
+  account: UUID
+  payee_name?: string
+}
+
+export interface ImportTransactionsResponse {
+  errors?: Message[]
+  added: number
+  updated: number
+}
+
+export interface Message {
+  message: string
+}

packages/sparebank1/db/queries.ts

@@ -1,22 +1,16 @@
 import Database from "better-sqlite3"
-
+import dayjs from "dayjs"
-import dayjs, { type Dayjs } from "dayjs"
 import type { OAuthTokenResponse } from "@sb1/types.ts"
-
+import type {
-export type TokenResponse = {
+  TokenKey,
-  key: TokenKey
+  TokenResponse,
-  token: string
+  TokenResponseRaw,
-  expires_at: Dayjs
+} from "@sb1impl/db/types.ts"
-}
+import logger from "@common/logger.ts"
-
-type TokenResponseRaw = {
-  [K in keyof TokenResponse]: K extends "expires_at" ? string : TokenResponse[K]
-}
-
-export type TokenKey = "access-token" | "refresh-token"
 
 export function createDb(filepath: string) {
   const db = new Database(filepath)
+  logger.info(`Started Sqlite database at '${filepath}'`)
   db.pragma("journal_mode = WAL")
   db.exec(
     "CREATE TABLE IF NOT EXISTS tokens ('key' VARCHAR PRIMARY KEY, token VARCHAR NOT NULL, expires_at DATETIME NOT NULL)",

packages/sparebank1/db/types.ts

@@ -0,0 +1,13 @@
+import type { Dayjs } from "dayjs"
+
+export type TokenResponse = {
+  key: TokenKey
+  token: string
+  expires_at: Dayjs
+}
+
+export type TokenResponseRaw = {
+  [K in keyof TokenResponse]: K extends "expires_at" ? string : TokenResponse[K]
+}
+
+export type TokenKey = "access-token" | "refresh-token"

packages/sparebank1/mappings.ts

@@ -1,10 +1,10 @@
-import type { UUID } from "node:crypto"
-import dayjs from "dayjs"
-import { type ActualTransaction } from "@/actual.ts"
 import { ACTUAL_ACCOUNT_IDS, BANK_ACCOUNT_IDS } from "@/config.ts"
-import logger from "@common/logger.ts"
 import { toISODateString } from "@common/date.ts"
+import dayjs from "dayjs"
+
+import type { ActualTransaction } from "@common/types.ts"
 import type { SB1Transaction } from "@sb1/types.ts"
+import type { UUID } from "node:crypto"
 
 export function bankTransactionIntoActualTransaction(
   transaction: SB1Transaction,
@@ -18,7 +18,6 @@ export function bankTransactionIntoActualTransaction(
     amount: Math.floor(transaction.amount * 100),
     date: toISODateString(dayjs(transaction.date)),
     payee_name: transaction.cleanedDescription,
-    // TODO if not cleared or nonUniqueId is 0, rerun later
     cleared: isCleared(transaction),
   }
 }
@@ -38,9 +37,7 @@ function getActualAccountId(transcation: SB1Transaction): UUID {
       return ACTUAL_ACCOUNT_IDS[i] as UUID
     }
   }
-  const error = new Error(
+  throw new Error(
     "Failed to find ActualAccountId, length of BANK_ACCOUNT_IDS and ACTUAL_ACCOUNT_IDS must match",
   )
-  logger.error(error)
-  throw error
 }

packages/sparebank1/sparebank1.ts

@@ -1,42 +1,38 @@
+import * as Transactions from "@sb1/transactions.ts"
+import * as Oauth from "@sb1/oauth.ts"
 import {
   BANK_INITIAL_REFRESH_TOKEN,
   BANK_OAUTH_CLIENT_ID,
   BANK_OAUTH_CLIENT_SECRET,
+  DB_DIRECTORY,
+  DB_FILENAME,
 } from "@/config.ts"
-import logger from "@common/logger.ts"
-import dayjs, { type Dayjs } from "dayjs"
-import type { Database } from "better-sqlite3"
 import {
   clearTokens,
+  createDb,
   fetchToken,
   insertTokens,
-  type TokenResponse,
+} from "@sb1impl/db/queries.ts"
-} from "@/bank/db/queries.ts"
+import { bankTransactionIntoActualTransaction } from "./mappings.ts"
-import * as Oauth from "@sb1/oauth.ts"
+import { createDirIfMissing } from "@/fs.ts"
-import * as Transactions from "@sb1/transactions.ts"
+import logger from "@common/logger.ts"
-import type { ActualTransaction } from "@/actual.ts"
+import dayjs from "dayjs"
-import { bankTransactionIntoActualTransaction } from "@/mappings.ts"
+
+import type { ActualTransaction, Bank, Interval } from "@common/types.ts"
+import type { TokenResponse } from "@sb1impl/db/types.ts"
 import type { OAuthTokenResponse } from "@sb1/types.ts"
-
+import type { Database } from "better-sqlite3"
-export interface Bank {
-  fetchTransactions: (
-    interval: Interval,
-    ...accountKeys: ReadonlyArray<string>
-  ) => Promise<ReadonlyArray<ActualTransaction>>
-}
-
-export interface Interval {
-  fromDate: Dayjs
-  toDate: Dayjs
-}
 
 export class Sparebank1Impl implements Bank {
   private readonly db: Database
 
-  constructor(db: Database) {
+  constructor() {
-    this.db = db
+    createDirIfMissing(DB_DIRECTORY)
+    const databaseFilePath = `${DB_DIRECTORY}/${DB_FILENAME}.sqlite`
+    this.db = createDb(databaseFilePath)
   }
 
+  // TODO if not cleared rerun later
   async fetchTransactions(
     interval: Interval,
     ...accountKeys: ReadonlyArray<string>
@@ -46,8 +42,19 @@ export class Sparebank1Impl implements Bank {
       accountKeys,
       interval,
     )
-    const sparebankTransactions = response.transactions
+
-    return sparebankTransactions.map(bankTransactionIntoActualTransaction)
+    return response.transactions
+      .map((transaction) => {
+        const actualTransaction =
+          bankTransactionIntoActualTransaction(transaction)
+
+        return actualTransaction.cleared === true ? actualTransaction : null
+      })
+      .filter((transaction) => transaction !== null)
+  }
+
+  shutdown(): void {
+    this.db.close()
   }
 
   private async getAccessToken(): Promise<string> {

packages/sparebank1Api/oauth.ts

@@ -1,4 +1,4 @@
-import { OAuthTokenResponse, Result } from "./types"
+import type { OAuthTokenResponse, Result } from "./types"
 import * as querystring from "node:querystring"
 import { baseUrl, failure, success } from "./common"
 import logger from "@common/logger"

packages/sparebank1Api/transactions.ts

@@ -1,8 +1,9 @@
-import type { Interval, TransactionResponse } from "./types"
+import type { TransactionResponse } from "./types"
 import * as querystring from "node:querystring"
 import { toISODateString } from "@common/date"
 import logger from "@common/logger"
 import { baseUrl } from "./common"
+import type { Interval } from "@common/types.ts"
 
 export async function list(
   accessToken: string,
@@ -18,7 +19,7 @@ export async function list(
   })
 
   const url = `${baseUrl}/personal/banking/transactions?${queryString}`
-  logger.debug(`Sending GET request to '${url}'`)
+  logger.info(`GET '${url}'`)
   const response = await fetch(url, {
     headers: {
       Authorization: `Bearer ${accessToken}`,

packages/sparebank1Api/types.ts

@@ -1,14 +1,7 @@
-import type { Dayjs } from "dayjs"
-
 export type Success<T> = { status: "success"; data: T }
 export type Failure<T> = { status: "failure"; data: T }
 export type Result<OK, Err> = Success<OK> | Failure<Err>
 
-export interface Interval {
-  fromDate: Dayjs
-  toDate: Dayjs
-}
-
 export interface OAuthTokenResponse {
   access_token: string
   expires_in: number
@@ -20,6 +13,10 @@ export interface OAuthTokenResponse {
 
 export type BookingStatus = "PENDING" | "BOOKED"
 
+/**
+ * 18-character unique ID used to identify a transaction
+ * The value is "000000000000000000" until the transaction is booked, and might be set a few days later
+ */
 export type NonUniqueId = "000000000000000000" | `${number}`
 
 export interface SB1Transaction {

src/actual.ts

@@ -5,38 +5,20 @@ import {
   ACTUAL_SERVER_URL,
   ACTUAL_SYNC_ID,
 } from "@/config.ts"
-import type { TransactionEntity } from "@actual-app/api/@types/loot-core/types/models"
-import { type UUID } from "node:crypto"
 import logger from "@common/logger.ts"
-
+import type { UUID } from "node:crypto"
-export interface Actual {
+import type {
-  importTransactions: (
+  Actual,
-    accountId: UUID,
+  ActualTransaction,
-    transactions: Iterable<ActualTransaction>,
+  ImportTransactionsResponse,
-  ) => Promise<ImportTransactionsResponse>
+} from "@common/types.ts"
-
+import { createDirIfMissing } from "@/fs.ts"
-  shutdown: () => Promise<void>
-}
-
-export interface ActualTransaction extends TransactionEntity {
-  account: UUID
-  payee_name?: string
-}
-
-export interface Message {
-  message: string
-}
-
-export interface ImportTransactionsResponse {
-  errors?: Message[]
-  added: number
-  updated: number
-}
 
 export class ActualImpl implements Actual {
   private constructor() {}
 
   static async init(): Promise<Actual> {
+    createDirIfMissing(ACTUAL_DATA_DIR)
     await actual.init({
       // Budget data will be cached locally here, in subdirectories for each file.
       dataDir: ACTUAL_DATA_DIR,
@@ -50,6 +32,21 @@ export class ActualImpl implements Actual {
     return new ActualImpl()
   }
 
+  /**
+   * Attempts to connect then immediatly shutsdown the connection
+   * @exception error If connection fails
+   */
+  static async testConnection(): Promise<void> {
+    let actual: Actual | undefined
+    logger.info("Testing ActualBudget connection")
+    try {
+      actual = await ActualImpl.init()
+    } finally {
+      await actual?.shutdown()
+      logger.info("Finished testing ActualBudget connection")
+    }
+  }
+
   async importTransactions(
     accountId: UUID,
     transactions: Iterable<ActualTransaction>,
@@ -58,6 +55,7 @@ export class ActualImpl implements Actual {
   }
 
   async shutdown(): Promise<void> {
+    logger.info(`Shutting down ActualBudget API for ${ACTUAL_SERVER_URL}`)
     return actual.shutdown()
   }
 

src/fs.ts

@@ -1,10 +1,7 @@
 import * as fs from "node:fs"
 import logger from "@common/logger"
 
-export function createDirsIfMissing(...directories: string[]): void {
+// TODO move to common
-  directories.forEach(createDirIfMissing)
-}
-
 export function createDirIfMissing(directory: string): void {
   if (!fs.existsSync(directory)) {
     logger.info(`Missing '${directory}', creating...`)

src/main.ts

@@ -1,35 +1,47 @@
-import { type Actual, ActualImpl } from "@/actual.ts"
+import { ActualImpl } from "@/actual.ts"
 import { cronJobDaily } from "@/cron.ts"
-import { type Bank, type Interval, Sparebank1Impl } from "@/bank/sparebank1.ts"
 import {
-  ACTUAL_DATA_DIR,
   BANK_ACCOUNT_IDS,
-  DB_DIRECTORY,
-  DB_FILENAME,
   TRANSACTION_RELATIVE_FROM_DATE,
   TRANSACTION_RELATIVE_TO_DATE,
 } from "@/config.ts"
 import logger from "@common/logger.ts"
 import type { UUID } from "node:crypto"
-import { createDb } from "@/bank/db/queries.ts"
 import { CronJob } from "cron"
-import { createDirsIfMissing } from "@/fs.ts"
 import dayjs from "dayjs"
+import type { Actual, Bank, Interval } from "@common/types.ts"
+import { Sparebank1Impl } from "@sb1impl/sparebank1.ts"
 
 // TODO move tsx to devDependency. Requires ts support for Node with support for @ alias
 // TODO verbatimSyntax in tsconfig, conflicts with jest
-// TODO multi module project. Main | DAL | Sparebank1 impl
 // TODO store last fetched date in db, and refetch from that date, if app has been offline for some time
-// TODO do not fetch if saturday or sunday
 
-export async function daily(actual: Actual, bank: Bank): Promise<void> {
+async function main(): Promise<void> {
-  // Fetch transactions from the bank
+  logger.info("Starting application")
+  const bank = new Sparebank1Impl()
+
+  if (process.env.ONCE) {
+    return await runOnce(bank)
+  }
+  await ActualImpl.testConnection()
+  await runCronJob(bank)
+}
+
+export async function moveTransactions(
+  actual: Actual,
+  bank: Bank,
+): Promise<void> {
   const actualTransactions = await bank.fetchTransactions(
     relativeInterval(),
     ...BANK_ACCOUNT_IDS,
   )
   logger.info(`Fetched ${actualTransactions.length} transactions`)
 
+  if (actualTransactions.length === 0) {
+    logger.debug("No transactions to import, skipping")
+    return
+  }
+
   const transactionsByAccount = Object.groupBy(
     actualTransactions,
     (transaction) => transaction.account,
@@ -58,46 +70,58 @@ function relativeInterval(): Interval {
   }
 }
 
-async function main(): Promise<void> {
+async function runOnce(bank: Bank) {
-  logger.info("Starting application")
-
-  createDirsIfMissing(ACTUAL_DATA_DIR, DB_DIRECTORY)
-
   const actual = await ActualImpl.init()
-  const databaseFilePath = `${DB_DIRECTORY}/${DB_FILENAME}.sqlite`
-  const db = createDb(databaseFilePath)
-  logger.info(`Started Sqlite database at '${databaseFilePath}'`)
-  const bank = new Sparebank1Impl(db)
 
-  process.on("SIGINT", async () => {
+  registerInterrupt(bank)
-    logger.info("Caught interrupt signal")
-    await shutdown()
-  })
 
-  let cronJob: CronJob | undefined
-  if (process.env.ONCE) {
   try {
-      return await daily(actual, bank)
+    return await moveTransactions(actual, bank)
   } finally {
-      await shutdown()
-    }
-  }
-
-  logger.info("Waiting for CRON job to start")
-  // TODO init and shutdown resources when job runs?
-  try {
-    cronJob = cronJobDaily(async () => await daily(actual, bank))
-  } catch (exception) {
-    logger.error(exception, "Caught exception at daily job, shutting down!")
-    await shutdown()
-  }
-
-  async function shutdown(): Promise<void> {
-    logger.info("Shutting down, Bye!")
     await actual.shutdown()
-    db.close()
+    await shutdown(bank)
-    cronJob?.stop()
   }
 }
 
+async function runCronJob(bank: Bank): Promise<void> {
+  logger.info("Waiting for CronJob to start")
+
+  const cronJob = cronJobDaily(async () => {
+    let actual: Actual | undefined
+    try {
+      actual = await ActualImpl.init()
+      await moveTransactions(actual, bank)
+    } catch (exception) {
+      logger.error(exception, "Caught exception at CronJob, shutting down!")
+      await shutdown(bank, cronJob)
+    } finally {
+      await actual?.shutdown()
+    }
+  })
+  registerInterrupt(bank, cronJob)
+}
+
+let isShuttingDown = false
+
+function registerInterrupt(
+  bank: Bank,
+  cronJob: CronJob | undefined = undefined,
+): void {
+  process.on("SIGINT", async () => {
+    if (isShuttingDown) return
+    isShuttingDown = true
+    logger.info("Caught interrupt signal")
+    await shutdown(bank, cronJob)
+  })
+}
+
+async function shutdown(
+  bank: Bank,
+  cronJob: CronJob | undefined = undefined,
+): Promise<void> {
+  logger.info("Shutting down, Bye!")
+  await bank.shutdown()
+  cronJob?.stop()
+}
+
 void main()

tests/main.test.ts

@@ -1,6 +1,6 @@
 import { describe, it } from "@jest/globals"
 
-import { daily } from "@/main.ts"
+import { moveTransactions } from "@/main.ts"
 import { ActualImpl } from "@/actual.ts"
 import { BankStub } from "./stubs/bankStub.ts"
 
@@ -10,7 +10,7 @@ import { BankStub } from "./stubs/bankStub.ts"
 describe("Main logic of the application", () => {
   it("should import the transactions to Actual Budget", async () => {
     const actual = await ActualImpl.init()
-    await daily(actual, new BankStub())
+    await moveTransactions(actual, new BankStub())
     await actual.shutdown()
   })
 })

tsconfig.json

@@ -13,7 +13,8 @@
     "paths": {
       "@/*": ["./src/*"],
       "@common/*": ["./packages/common/*"],
-      "@sb1/*": ["./packages/sparebank1Api/*"]
+      "@sb1/*": ["./packages/sparebank1Api/*"],
+      "@sb1impl/*": ["./packages/sparebank1/*"]
     }
   },
   "exclude": ["node_modules", "./*.ts", "__test__"]