Compare commits

..

No commits in common. "95ddcbaf13241e5852fa1cff4df66574941d8b95" and "06cc89f76279d6b6d59d30e9194badc98b229b46" have entirely different histories.

13 changed files with 128 additions and 188 deletions

View File

@ -1,7 +1,7 @@
services: services:
server: server:
container_name: actual_sparebank1_cronjob container_name: actual_sparebank1_cronjob
restart: unless-stopped restart: no
build: build:
context: . context: .
environment: environment:
@ -16,7 +16,7 @@ services:
- BANK_OAUTH_CLIENT_SECRET - BANK_OAUTH_CLIENT_SECRET
- BANK_ACCOUNT_IDS - BANK_ACCOUNT_IDS
- LOG_LEVEL - LOG_LEVEL
- DB_DIRECTORY - DB_DIRECTORY # Required for Docker Compose
- DB_FILENAME - DB_FILENAME
- TRANSACTION_RELATIVE_FROM_DATE - TRANSACTION_RELATIVE_FROM_DATE
- TRANSACTION_RELATIVE_TO_DATE - TRANSACTION_RELATIVE_TO_DATE
@ -24,7 +24,6 @@ services:
- cache:/${ACTUAL_DATA_DIR:-.cache} - cache:/${ACTUAL_DATA_DIR:-.cache}
- data:/${DB_DIRECTORY:-data} - data:/${DB_DIRECTORY:-data}
# TODO change volume name from hostexecutor-*
volumes: volumes:
cache: cache:
data: data:

View File

@ -1,66 +0,0 @@
import type { Dayjs } from "dayjs"
import type { UUID } from "node:crypto"
import type { TransactionEntity } from "@actual-app/api/@types/loot-core/types/models"
/**
* Defines how to interact with the bank
*/
export interface Bank {
/**
* Fetch all transactions in the specified days, from the given accounts
* @param interval Which days to fetch transactions for
* @param accountKeys The id of the accounts to fetch transactions from
* @returns An array of all transactions
*/
fetchTransactions: (
interval: Interval,
...accountKeys: ReadonlyArray<string>
) => Promise<ReadonlyArray<ActualTransaction>>
/**
* Shutdown resources
*/
shutdown: () => Promise<void> | void
}
export interface Interval {
fromDate: Dayjs
toDate: Dayjs
}
/**
* Describes how to interact with ActualBudget
*/
export interface Actual {
/**
* Import transactions following the rules defined in the ActualBudget instance
* If the transactions exists, it will be updated, or no change should be done.
* @param accountId The ActualBudget id to upload to
* @param transactions The transactions to import
* @returns An object describing what changed
*/
importTransactions: (
accountId: UUID,
transactions: Iterable<ActualTransaction>,
) => Promise<ImportTransactionsResponse>
/**
* Disconnect from ActualBudget and release resources
*/
shutdown: () => Promise<void>
}
export interface ActualTransaction extends TransactionEntity {
account: UUID
payee_name?: string
}
export interface ImportTransactionsResponse {
errors?: Message[]
added: number
updated: number
}
export interface Message {
message: string
}

View File

@ -1,13 +0,0 @@
import type { Dayjs } from "dayjs"
export type TokenResponse = {
key: TokenKey
token: string
expires_at: Dayjs
}
export type TokenResponseRaw = {
[K in keyof TokenResponse]: K extends "expires_at" ? string : TokenResponse[K]
}
export type TokenKey = "access-token" | "refresh-token"

View File

@ -1,4 +1,4 @@
import type { OAuthTokenResponse, Result } from "./types" import { OAuthTokenResponse, Result } from "./types"
import * as querystring from "node:querystring" import * as querystring from "node:querystring"
import { baseUrl, failure, success } from "./common" import { baseUrl, failure, success } from "./common"
import logger from "@common/logger" import logger from "@common/logger"

View File

@ -1,9 +1,8 @@
import type { TransactionResponse } from "./types" import type { Interval, TransactionResponse } from "./types"
import * as querystring from "node:querystring" import * as querystring from "node:querystring"
import { toISODateString } from "@common/date" import { toISODateString } from "@common/date"
import logger from "@common/logger" import logger from "@common/logger"
import { baseUrl } from "./common" import { baseUrl } from "./common"
import type { Interval } from "@common/types.ts"
export async function list( export async function list(
accessToken: string, accessToken: string,
@ -19,7 +18,7 @@ export async function list(
}) })
const url = `${baseUrl}/personal/banking/transactions?${queryString}` const url = `${baseUrl}/personal/banking/transactions?${queryString}`
logger.info(`GET '${url}'`) logger.debug(`Sending GET request to '${url}'`)
const response = await fetch(url, { const response = await fetch(url, {
headers: { headers: {
Authorization: `Bearer ${accessToken}`, Authorization: `Bearer ${accessToken}`,

View File

@ -1,7 +1,14 @@
import type { Dayjs } from "dayjs"
export type Success<T> = { status: "success"; data: T } export type Success<T> = { status: "success"; data: T }
export type Failure<T> = { status: "failure"; data: T } export type Failure<T> = { status: "failure"; data: T }
export type Result<OK, Err> = Success<OK> | Failure<Err> export type Result<OK, Err> = Success<OK> | Failure<Err>
export interface Interval {
fromDate: Dayjs
toDate: Dayjs
}
export interface OAuthTokenResponse { export interface OAuthTokenResponse {
access_token: string access_token: string
expires_in: number expires_in: number
@ -13,10 +20,6 @@ export interface OAuthTokenResponse {
export type BookingStatus = "PENDING" | "BOOKED" export type BookingStatus = "PENDING" | "BOOKED"
/**
* 18-character unique ID used to identify a transaction
* The value is "000000000000000000" until the transaction is booked, and might be set a few days later
*/
export type NonUniqueId = "000000000000000000" | `${number}` export type NonUniqueId = "000000000000000000" | `${number}`
export interface SB1Transaction { export interface SB1Transaction {

View File

@ -5,20 +5,38 @@ import {
ACTUAL_SERVER_URL, ACTUAL_SERVER_URL,
ACTUAL_SYNC_ID, ACTUAL_SYNC_ID,
} from "@/config.ts" } from "@/config.ts"
import type { TransactionEntity } from "@actual-app/api/@types/loot-core/types/models"
import { type UUID } from "node:crypto"
import logger from "@common/logger.ts" import logger from "@common/logger.ts"
import type { UUID } from "node:crypto"
import type { export interface Actual {
Actual, importTransactions: (
ActualTransaction, accountId: UUID,
ImportTransactionsResponse, transactions: Iterable<ActualTransaction>,
} from "@common/types.ts" ) => Promise<ImportTransactionsResponse>
import { createDirIfMissing } from "@/fs.ts"
shutdown: () => Promise<void>
}
export interface ActualTransaction extends TransactionEntity {
account: UUID
payee_name?: string
}
export interface Message {
message: string
}
export interface ImportTransactionsResponse {
errors?: Message[]
added: number
updated: number
}
export class ActualImpl implements Actual { export class ActualImpl implements Actual {
private constructor() {} private constructor() {}
static async init(): Promise<Actual> { static async init(): Promise<Actual> {
createDirIfMissing(ACTUAL_DATA_DIR)
await actual.init({ await actual.init({
// Budget data will be cached locally here, in subdirectories for each file. // Budget data will be cached locally here, in subdirectories for each file.
dataDir: ACTUAL_DATA_DIR, dataDir: ACTUAL_DATA_DIR,
@ -55,7 +73,6 @@ export class ActualImpl implements Actual {
} }
async shutdown(): Promise<void> { async shutdown(): Promise<void> {
logger.info(`Shutting down ActualBudget API for ${ACTUAL_SERVER_URL}`)
return actual.shutdown() return actual.shutdown()
} }

View File

@ -1,16 +1,22 @@
import Database from "better-sqlite3" import Database from "better-sqlite3"
import dayjs from "dayjs"
import dayjs, { type Dayjs } from "dayjs"
import type { OAuthTokenResponse } from "@sb1/types.ts" import type { OAuthTokenResponse } from "@sb1/types.ts"
import type {
TokenKey, export type TokenResponse = {
TokenResponse, key: TokenKey
TokenResponseRaw, token: string
} from "@sb1impl/db/types.ts" expires_at: Dayjs
import logger from "@common/logger.ts" }
type TokenResponseRaw = {
[K in keyof TokenResponse]: K extends "expires_at" ? string : TokenResponse[K]
}
export type TokenKey = "access-token" | "refresh-token"
export function createDb(filepath: string) { export function createDb(filepath: string) {
const db = new Database(filepath) const db = new Database(filepath)
logger.info(`Started Sqlite database at '${filepath}'`)
db.pragma("journal_mode = WAL") db.pragma("journal_mode = WAL")
db.exec( db.exec(
"CREATE TABLE IF NOT EXISTS tokens ('key' VARCHAR PRIMARY KEY, token VARCHAR NOT NULL, expires_at DATETIME NOT NULL)", "CREATE TABLE IF NOT EXISTS tokens ('key' VARCHAR PRIMARY KEY, token VARCHAR NOT NULL, expires_at DATETIME NOT NULL)",

View File

@ -2,33 +2,39 @@ import {
BANK_INITIAL_REFRESH_TOKEN, BANK_INITIAL_REFRESH_TOKEN,
BANK_OAUTH_CLIENT_ID, BANK_OAUTH_CLIENT_ID,
BANK_OAUTH_CLIENT_SECRET, BANK_OAUTH_CLIENT_SECRET,
DB_DIRECTORY,
DB_FILENAME,
} from "@/config.ts" } from "@/config.ts"
import logger from "@common/logger.ts" import logger from "@common/logger.ts"
import dayjs from "dayjs" import dayjs, { type Dayjs } from "dayjs"
import type { Database } from "better-sqlite3" import type { Database } from "better-sqlite3"
import { import {
clearTokens, clearTokens,
createDb,
fetchToken, fetchToken,
insertTokens, insertTokens,
} from "@sb1impl/db/queries.ts" type TokenResponse,
} from "@/bank/db/queries.ts"
import * as Oauth from "@sb1/oauth.ts" import * as Oauth from "@sb1/oauth.ts"
import * as Transactions from "@sb1/transactions.ts" import * as Transactions from "@sb1/transactions.ts"
import { bankTransactionIntoActualTransaction } from "./mappings.ts" import type { ActualTransaction } from "@/actual.ts"
import { bankTransactionIntoActualTransaction } from "@/mappings.ts"
import type { OAuthTokenResponse } from "@sb1/types.ts" import type { OAuthTokenResponse } from "@sb1/types.ts"
import type { ActualTransaction, Bank, Interval } from "@common/types.ts"
import type { TokenResponse } from "@sb1impl/db/types.ts" export interface Bank {
import { createDirIfMissing } from "@/fs.ts" fetchTransactions: (
interval: Interval,
...accountKeys: ReadonlyArray<string>
) => Promise<ReadonlyArray<ActualTransaction>>
}
export interface Interval {
fromDate: Dayjs
toDate: Dayjs
}
export class Sparebank1Impl implements Bank { export class Sparebank1Impl implements Bank {
private readonly db: Database private readonly db: Database
constructor() { constructor(db: Database) {
createDirIfMissing(DB_DIRECTORY) this.db = db
const databaseFilePath = `${DB_DIRECTORY}/${DB_FILENAME}.sqlite`
this.db = createDb(databaseFilePath)
} }
async fetchTransactions( async fetchTransactions(
@ -44,10 +50,6 @@ export class Sparebank1Impl implements Bank {
return sparebankTransactions.map(bankTransactionIntoActualTransaction) return sparebankTransactions.map(bankTransactionIntoActualTransaction)
} }
shutdown(): void {
this.db.close()
}
private async getAccessToken(): Promise<string> { private async getAccessToken(): Promise<string> {
const accessToken = fetchToken(this.db, "access-token") const accessToken = fetchToken(this.db, "access-token")

View File

@ -1,6 +1,10 @@
import * as fs from "node:fs" import * as fs from "node:fs"
import logger from "@common/logger" import logger from "@common/logger"
export function createDirsIfMissing(...directories: string[]): void {
directories.forEach(createDirIfMissing)
}
export function createDirIfMissing(directory: string): void { export function createDirIfMissing(directory: string): void {
if (!fs.existsSync(directory)) { if (!fs.existsSync(directory)) {
logger.info(`Missing '${directory}', creating...`) logger.info(`Missing '${directory}', creating...`)

View File

@ -1,37 +1,32 @@
import { ActualImpl } from "@/actual.ts" import { type Actual, ActualImpl } from "@/actual.ts"
import { cronJobDaily } from "@/cron.ts" import { cronJobDaily } from "@/cron.ts"
import { type Bank, type Interval, Sparebank1Impl } from "@/bank/sparebank1.ts"
import { import {
ACTUAL_DATA_DIR,
BANK_ACCOUNT_IDS, BANK_ACCOUNT_IDS,
DB_DIRECTORY,
DB_FILENAME,
TRANSACTION_RELATIVE_FROM_DATE, TRANSACTION_RELATIVE_FROM_DATE,
TRANSACTION_RELATIVE_TO_DATE, TRANSACTION_RELATIVE_TO_DATE,
} from "@/config.ts" } from "@/config.ts"
import logger from "@common/logger.ts" import logger from "@common/logger.ts"
import type { UUID } from "node:crypto" import type { UUID } from "node:crypto"
import { createDb } from "@/bank/db/queries.ts"
import { CronJob } from "cron" import { CronJob } from "cron"
import { createDirsIfMissing } from "@/fs.ts"
import dayjs from "dayjs" import dayjs from "dayjs"
import type { Actual, Bank, Interval } from "@common/types.ts"
import { Sparebank1Impl } from "@sb1impl/sparebank1.ts"
// TODO move tsx to devDependency. Requires ts support for Node with support for @ alias // TODO move tsx to devDependency. Requires ts support for Node with support for @ alias
// TODO verbatimSyntax in tsconfig, conflicts with jest // TODO verbatimSyntax in tsconfig, conflicts with jest
// TODO multi module project. Main | DAL | Sparebank1 impl
// TODO store last fetched date in db, and refetch from that date, if app has been offline for some time // TODO store last fetched date in db, and refetch from that date, if app has been offline for some time
// TODO do not fetch if saturday or sunday // TODO do not fetch if saturday or sunday
async function main(): Promise<void> {
logger.info("Starting application")
const bank = new Sparebank1Impl()
if (process.env.ONCE) {
return await runOnce(bank)
}
await ActualImpl.testConnection()
await runCronJob(bank)
}
export async function moveTransactions( export async function moveTransactions(
actual: Actual, actual: Actual,
bank: Bank, bank: Bank,
): Promise<void> { ): Promise<void> {
// Fetch transactions from the bank
const actualTransactions = await bank.fetchTransactions( const actualTransactions = await bank.fetchTransactions(
relativeInterval(), relativeInterval(),
...BANK_ACCOUNT_IDS, ...BANK_ACCOUNT_IDS,
@ -66,58 +61,53 @@ function relativeInterval(): Interval {
} }
} }
async function runOnce(bank: Bank) { async function main(): Promise<void> {
logger.info("Starting application")
createDirsIfMissing(ACTUAL_DATA_DIR, DB_DIRECTORY)
const databaseFilePath = `${DB_DIRECTORY}/${DB_FILENAME}.sqlite`
const db = createDb(databaseFilePath)
logger.info(`Started Sqlite database at '${databaseFilePath}'`)
const bank = new Sparebank1Impl(db)
process.on("SIGINT", async () => {
logger.info("Caught interrupt signal")
await shutdown()
})
let cronJob: CronJob | undefined
if (process.env.ONCE) {
const actual = await ActualImpl.init() const actual = await ActualImpl.init()
registerInterrupt(bank)
try { try {
return await moveTransactions(actual, bank) return await moveTransactions(actual, bank)
} finally { } finally {
await actual.shutdown() await actual.shutdown()
await shutdown(bank) await shutdown()
} }
} else {
await ActualImpl.testConnection()
} }
async function runCronJob(bank: Bank): Promise<void> {
logger.info("Waiting for CronJob to start") logger.info("Waiting for CronJob to start")
const cronJob = cronJobDaily(async () => {
let actual: Actual | undefined let actual: Actual | undefined
try { try {
cronJob = cronJobDaily(async () => {
actual = await ActualImpl.init() actual = await ActualImpl.init()
await moveTransactions(actual, bank) await moveTransactions(actual, bank)
})
} catch (exception) { } catch (exception) {
logger.error(exception, "Caught exception at CronJob, shutting down!") logger.error(exception, "Caught exception at CronJob, shutting down!")
await shutdown(bank, cronJob) await shutdown()
} finally { } finally {
await actual?.shutdown() await actual?.shutdown()
} }
})
registerInterrupt(bank, cronJob)
}
let isShuttingDown = false async function shutdown(): Promise<void> {
function registerInterrupt(
bank: Bank,
cronJob: CronJob | undefined = undefined,
): void {
process.on("SIGINT", async () => {
if (isShuttingDown) return
isShuttingDown = true
logger.info("Caught interrupt signal")
await shutdown(bank, cronJob)
})
}
async function shutdown(
bank: Bank,
cronJob: CronJob | undefined = undefined,
): Promise<void> {
logger.info("Shutting down, Bye!") logger.info("Shutting down, Bye!")
await bank.shutdown() db.close()
cronJob?.stop() cronJob?.stop()
} }
}
void main() void main()

View File

@ -1,10 +1,10 @@
import type { UUID } from "node:crypto" import type { UUID } from "node:crypto"
import dayjs from "dayjs" import dayjs from "dayjs"
import { type ActualTransaction } from "@/actual.ts"
import { ACTUAL_ACCOUNT_IDS, BANK_ACCOUNT_IDS } from "@/config.ts" import { ACTUAL_ACCOUNT_IDS, BANK_ACCOUNT_IDS } from "@/config.ts"
import logger from "@common/logger.ts" import logger from "@common/logger.ts"
import { toISODateString } from "@common/date.ts" import { toISODateString } from "@common/date.ts"
import type { SB1Transaction } from "@sb1/types.ts" import type { SB1Transaction } from "@sb1/types.ts"
import type { ActualTransaction } from "@common/types.ts"
export function bankTransactionIntoActualTransaction( export function bankTransactionIntoActualTransaction(
transaction: SB1Transaction, transaction: SB1Transaction,

View File

@ -13,8 +13,7 @@
"paths": { "paths": {
"@/*": ["./src/*"], "@/*": ["./src/*"],
"@common/*": ["./packages/common/*"], "@common/*": ["./packages/common/*"],
"@sb1/*": ["./packages/sparebank1Api/*"], "@sb1/*": ["./packages/sparebank1Api/*"]
"@sb1impl/*": ["./packages/sparebank1/*"]
} }
}, },
"exclude": ["node_modules", "./*.ts", "__test__"] "exclude": ["node_modules", "./*.ts", "__test__"]