Internal state persistence (#27)

Allow us to generate year over year reports without having to rerun everything from the beginning. Co-authored-by: Natercio Moniz <[email protected]>
2026-08-03 00:33:06 +01:00
parent 1ce8561782
commit 9bd4230ff1
25 changed files with 1726 additions and 178 deletions
@@ -0,0 +1,159 @@
+package ofigi
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/biter777/countries"
+	"golang.org/x/time/rate"
+)
+
+var OpenFIGIAPIKeyHeader = http.CanonicalHeaderKey("X-OPENFIGI-APIKEY")
+
+// Client is a thin adapter for the openfigi.com api.
+type Client struct {
+	client         *http.Client
+	apiKey         string
+	mappingLimiter *rate.Limiter
+
+	mu sync.RWMutex
+	// TODO: there's no eviction policy at the moment as this is only used by short-lived application
+	// which processes a relatively small amount of records. We need to consider using an external
+	// cache lib (like golang-lru or go-cache) if this becomes a problem or implement eviction
+	// ourselves.
+	securityTypeCache map[string]string
+}
+
+// NewOpenFIGI creates an OpenFIGI client that uses the API key if provided
+func NewOpenFIGI(c *http.Client, apiKey string) *Client {
+	// Rate limits as per https://www.openfigi.com/api/documentation#rate-limits
+	limiter := rate.NewLimiter(rate.Every(time.Minute), 25)
+	if len(apiKey) > 0 {
+		slog.Debug("OpenFIGI client: created with API Key rate limits")
+		limiter = rate.NewLimiter(rate.Every(time.Second*6), 25)
+	} else {
+		slog.Debug("OpenFIGI client: created with puplic rate limits")
+	}
+
+	return &Client{
+		client:            c,
+		apiKey:            apiKey,
+		mappingLimiter:    limiter,
+		securityTypeCache: make(map[string]string),
+	}
+}
+
+func (of *Client) SecurityTypeByISIN(ctx context.Context, isin string) (string, error) {
+	of.mu.RLock()
+	if secType, ok := of.securityTypeCache[isin]; ok {
+		of.mu.RUnlock()
+		slog.Debug("OpenFIGI client: SecurityTypeByISIN cache hit",
+			slog.String("isin", isin),
+			slog.String("security_type", secType))
+		return secType, nil
+	}
+	of.mu.RUnlock()
+
+	slog.Debug("OpenFIGI client: SecurityTypeByISIN cache miss",
+		slog.String("isin", isin))
+
+	of.mu.Lock()
+	defer of.mu.Unlock()
+
+	// we check again because there could be more than one concurrent cache miss and we want only one
+	// of them to result in an actual request. When the first one releases the lock the following
+	// reads will hit the cache.
+	if secType, ok := of.securityTypeCache[isin]; ok {
+		return secType, nil
+	}
+
+	if len(isin) != 12 || countries.ByName(isin[:2]) == countries.Unknown {
+		return "", fmt.Errorf("invalid ISIN: %s", isin)
+	}
+
+	rawBody, err := json.Marshal([]mappingRequestBody{{
+		IDType:  "ID_ISIN",
+		IDValue: isin,
+	}})
+	if err != nil {
+		return "", fmt.Errorf("marshal mapping request body: %w", err)
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, "https://api.openfigi.com/v3/mapping", bytes.NewBuffer(rawBody))
+	if err != nil {
+		return "", fmt.Errorf("create mapping request: %w", err)
+	}
+
+	req.Header.Add("Content-Type", "application/json")
+
+	if len(of.apiKey) > 0 {
+		req.Header.Add(OpenFIGIAPIKeyHeader, of.apiKey)
+	}
+
+	if !of.mappingLimiter.Allow() {
+		slog.Debug("OpenFIGI client: mapping limiter waiting for rate limiter capacity")
+	}
+
+	err = of.mappingLimiter.Wait(ctx)
+	if err != nil {
+		return "", fmt.Errorf("wait for mapping request capacity: %w", err)
+	}
+
+	res, err := of.client.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("make mapping request: %w", err)
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode >= 400 {
+		return "", fmt.Errorf("bad mapping response status code: %s", res.Status)
+	}
+
+	var resBody []mappingResponseBody
+	err = json.NewDecoder(res.Body).Decode(&resBody)
+	if err != nil {
+		return "", fmt.Errorf("unmarshal response: %w", err)
+	}
+
+	if len(resBody) == 0 {
+		return "", fmt.Errorf("missing top-level elements")
+	}
+
+	if len(resBody[0].Data) == 0 {
+		return "", fmt.Errorf("missing data elements")
+	}
+
+	// It is not possible that an isin is assign to different security types, therefore we can assume
+	// all entries have the same securityType value.
+	secType := resBody[0].Data[0].SecurityType
+	if secType == "" {
+		return "", fmt.Errorf("empty security type returned for ISIN: %s", isin)
+	}
+
+	of.securityTypeCache[isin] = secType
+
+	slog.Debug("OpenFIGI client: SecurityTypeByISIN cached mapping",
+		slog.String("isin", isin),
+		slog.String("security_type", secType))
+
+	return secType, nil
+}
+
+type mappingRequestBody struct {
+	IDType  string `json:"idType"`
+	IDValue string `json:"idValue"`
+}
+
+type mappingResponseBody struct {
+	Data []struct {
+		FIGI         string `json:"figi"`
+		SecurityType string `json:"securityType"`
+		Ticker       string `json:"ticker"`
+	} `json:"data"`
+}
@@ -0,0 +1,230 @@
+package ofigi_test
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/nmoniz/any2anexoj/internal/ofigi"
+)
+
+func TestOpenFIGI_SecurityTypeByISIN(t *testing.T) {
+	tests := []struct {
+		name    string // description of this test case
+		client  *http.Client
+		isin    string
+		want    string
+		wantErr bool
+	}{
+		{
+			name: "all good",
+			client: NewTestClient(t, func(req *http.Request) (*http.Response, error) {
+				return &http.Response{
+					Status:     http.StatusText(http.StatusOK),
+					StatusCode: http.StatusOK,
+					Body:       io.NopCloser(bytes.NewBufferString(`[{"data":[{"figi":"BBG000BJJR23","name":"AIRBUS SE","ticker":"EADSF","exchCode":"US","compositeFIGI":"BBG000BJJR23","securityType":"Common Stock","marketSector":"Equity","shareClassFIGI":"BBG001S8TFZ6","securityType2":"Common Stock","securityDescription":"EADSF"},{"figi":"BBG000BJJXJ2","name":"AIRBUS SE","ticker":"EADSF","exchCode":"PQ","compositeFIGI":"BBG000BJJR23","securityType":"Common Stock","marketSector":"Equity","shareClassFIGI":"BBG001S8TFZ6","securityType2":"Common Stock","securityDescription":"EADSF"}]}]`)),
+				}, nil
+			}),
+			isin: "NL0000235190",
+			want: "Common Stock",
+		},
+		{
+			name: "bad status code",
+			client: NewTestClient(t, func(req *http.Request) (*http.Response, error) {
+				return &http.Response{
+					Status:     http.StatusText(http.StatusTooManyRequests),
+					StatusCode: http.StatusTooManyRequests,
+				}, nil
+			}),
+			isin:    "NL0000235190",
+			wantErr: true,
+		},
+		{
+			name: "bad json",
+			client: NewTestClient(t, func(req *http.Request) (*http.Response, error) {
+				return &http.Response{
+					Status:     http.StatusText(http.StatusOK),
+					StatusCode: http.StatusOK,
+					Body:       io.NopCloser(bytes.NewBufferString(`{"bad": "json"}`)),
+				}, nil
+			}),
+			isin:    "NL0000235190",
+			wantErr: true,
+		},
+		{
+			name: "empty top-level",
+			client: NewTestClient(t, func(req *http.Request) (*http.Response, error) {
+				return &http.Response{
+					Status:     http.StatusText(http.StatusOK),
+					StatusCode: http.StatusOK,
+					Body:       io.NopCloser(bytes.NewBufferString(`[]`)),
+				}, nil
+			}),
+			isin:    "NL0000235190",
+			wantErr: true,
+		},
+		{
+			name: "empty data elements",
+			client: NewTestClient(t, func(req *http.Request) (*http.Response, error) {
+				return &http.Response{
+					Status:     http.StatusText(http.StatusOK),
+					StatusCode: http.StatusOK,
+					Body:       io.NopCloser(bytes.NewBufferString(`[{"data":[]}]`)),
+				}, nil
+			}),
+			isin:    "NL0000235190",
+			wantErr: true,
+		},
+		{
+			name: "empty securityType",
+			client: NewTestClient(t, func(req *http.Request) (*http.Response, error) {
+				return &http.Response{
+					Status:     http.StatusText(http.StatusOK),
+					StatusCode: http.StatusOK,
+					Body:       io.NopCloser(bytes.NewBufferString(`[{"data":[{"securityType":""}]}]`)),
+				}, nil
+			}),
+			isin:    "NL0000235190",
+			wantErr: true,
+		},
+		{
+			name: "client error",
+			client: NewTestClient(t, func(req *http.Request) (*http.Response, error) {
+				return nil, fmt.Errorf("boom")
+			}),
+			isin:    "NL0000235190",
+			wantErr: true,
+		},
+		{
+			name: "empty isin",
+			client: NewTestClient(t, func(req *http.Request) (*http.Response, error) {
+				t.Fatalf("should not make api request")
+				return nil, nil
+			}),
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			of := ofigi.NewOpenFIGI(tt.client, "")
+
+			got, gotErr := of.SecurityTypeByISIN(t.Context(), tt.isin)
+			if gotErr != nil {
+				if !tt.wantErr {
+					t.Errorf("want success but failed: %v", gotErr)
+				}
+				return
+			}
+			if tt.wantErr {
+				t.Fatal("want error but none")
+			}
+
+			if tt.want != got {
+				t.Fatalf("want security type to be %s but got %s", tt.want, got)
+			}
+		})
+	}
+}
+
+func TestOpenFIGI_SecurityTypeByISIN_Cache(t *testing.T) {
+	var alreadyCalled bool
+	c := NewTestClient(t, func(req *http.Request) (*http.Response, error) {
+		if alreadyCalled {
+			t.Fatalf("want requests to be cached")
+		}
+
+		alreadyCalled = true
+		return &http.Response{
+			Status:     http.StatusText(http.StatusOK),
+			StatusCode: http.StatusOK,
+			Body:       io.NopCloser(bytes.NewBufferString(`[{"data":[{"securityType":"Common Stock"}]}]`)),
+		}, nil
+	})
+
+	of := ofigi.NewOpenFIGI(c, "")
+
+	got, gotErr := of.SecurityTypeByISIN(t.Context(), "NL0000235190")
+	if gotErr != nil {
+		t.Fatalf("want 1st success call but got error: %v", gotErr)
+	}
+
+	if got != "Common Stock" {
+		t.Fatalf("want 1st securityType to be %q but got %q", "Common Stock", got)
+	}
+
+	got, gotErr = of.SecurityTypeByISIN(t.Context(), "NL0000235190")
+	if gotErr != nil {
+		t.Fatalf("want 2nd success call but got error: %v", gotErr)
+	}
+
+	if got != "Common Stock" {
+		t.Fatalf("want 2nd securityType to be %q but got %q", "Common Stock", got)
+	}
+}
+
+func TestOpenFIGI_SecurityTypeByISIN_APIKey(t *testing.T) {
+	t.Run("with API key", func(t *testing.T) {
+		wantAPIKey := "123abc-456xyz"
+
+		c := NewTestClient(t, func(req *http.Request) (*http.Response, error) {
+			value, ok := req.Header[ofigi.OpenFIGIAPIKeyHeader]
+			if !ok {
+				t.Fatalf("want %q header but got none: %v", ofigi.OpenFIGIAPIKeyHeader, req.Header)
+			}
+			if len(value) != 1 {
+				t.Fatalf("want exactly one %q header value but got %d", ofigi.OpenFIGIAPIKeyHeader, len(value))
+			}
+			if value[0] != wantAPIKey {
+				t.Fatalf("want %q header value %q but got %q", ofigi.OpenFIGIAPIKeyHeader, wantAPIKey, value[0])
+			}
+			return &http.Response{
+				Status:     http.StatusText(http.StatusOK),
+				StatusCode: http.StatusOK,
+				Body:       io.NopCloser(bytes.NewBufferString(`[{"data":[{"securityType":"Common Stock"}]}]`)),
+			}, nil
+		})
+		of := ofigi.NewOpenFIGI(c, wantAPIKey)
+
+		_, err := of.SecurityTypeByISIN(t.Context(), "US1234567890")
+		if err != nil {
+			t.Fatalf("want success but got an error: %s", err)
+		}
+	})
+
+	t.Run("without API key", func(t *testing.T) {
+		c := NewTestClient(t, func(req *http.Request) (*http.Response, error) {
+			_, ok := req.Header[ofigi.OpenFIGIAPIKeyHeader]
+			if ok {
+				t.Fatalf("want no %s header but got one", ofigi.OpenFIGIAPIKeyHeader)
+			}
+			return &http.Response{
+				Status:     http.StatusText(http.StatusOK),
+				StatusCode: http.StatusOK,
+				Body:       io.NopCloser(bytes.NewBufferString(`[{"data":[{"securityType":"Common Stock"}]}]`)),
+			}, nil
+		})
+		of := ofigi.NewOpenFIGI(c, "")
+		_, err := of.SecurityTypeByISIN(t.Context(), "US1234567890")
+		if err != nil {
+			t.Fatalf("want success but got an error: %s", err)
+		}
+	})
+}
+
+type RoundTripFunc func(req *http.Request) (*http.Response, error)
+
+func (f RoundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
+	return f(req)
+}
+
+func NewTestClient(t testing.TB, fn RoundTripFunc) *http.Client {
+	t.Helper()
+
+	return &http.Client{
+		Timeout:   time.Second,
+		Transport: fn,
+	}
+}