Author: Pierre-Luc Germain

# The IndexedFst class

# IndexedFst

The `IndexedFst` class provides fast named random access to indexed `fst` files. It is based on the fst package, which provides fast random reading of data frames. This is particularly useful to manipulate large collections of binding sites without loading them all in memory.

Creating an indexed fst file from a data.frame is very simple:

    library(scanMiRApp)

    ## Loading required package: scanMiR

    # we create a temporary directory in which the files will be saved
    tmp <- tempdir()
    f <- file.path(tmp, "test")
    # we create a dummy data.frame
    d <- data.frame( category=sample(LETTERS[1:4], 10000, replace=TRUE),
                     var2=sample(LETTERS, 10000, replace=TRUE),
                     var3=runif(10000) )

    saveIndexedFst(d, index.by="category", file.prefix=f)

The file can then be loaded (without having all the data in memory) in the following way:

    d2 <- loadIndexedFst(f)
    class(d2)

    ## [1] "IndexedFst"
    ## attr(,"package")
    ## [1] "scanMiRApp"

    summary(d2)

    ## <fst file>
    ## 10000 rows, 3 columns (test.fst)
    ##
    ## * 'category': character
    ## * 'var2'    : character
    ## * 'var3'    : double

We can see that `d2` is considerably smaller than the original `d`:

    format(object.size(d),units="Kb")

    ## [1] "237 Kb"

    format(object.size(d2),units="Kb")

    ## [1] "2.4 Kb"

Nevertheless, a number of functions can be used normally on the object:

    nrow(d2)

    ## [1] 10000

    ncol(d2)

    ## [1] 3

    colnames(d2)

    ## [1] "category" "var2"     "var3"

    head(d2)

    ##   category var2      var3
    ## 1        A    W 0.7106201
    ## 2        A    E 0.1466303
    ## 3        A    S 0.6228094
    ## 4        A    Y 0.4294806
    ## 5        A    S 0.2205327
    ## 6        A    P 0.1751266

In addition, the object can be accessed as a list (using the indexed variable). Since in this case the file is indexed using the category column, the different categories can be accessed as `names` of the object:

    names(d2)

    ## [1] "A" "B" "C" "D"

    lengths(d2)

    ##    A    B    C    D
    ## 2544 2419 2508 2529

We can read specifically the rows pertaining to one category using:

    catB <- d2$B
    head(catB)

    ##   category var2       var3
    ## 1        B    L 0.89117440
    ## 2        B    L 0.84840927
    ## 3        B    I 0.35597213
    ## 4        B    U 0.84521713
    ## 5        B    V 0.07683134
    ## 6        B    C 0.91938975

# Storing GRanges as IndexedFst

In addition to data.frames, GRanges can be saved as indexed Fst. To demonstrate this, we first create a dummy GRanges object:

    library(GenomicRanges)

    ## Loading required package: stats4

    ## Loading required package: BiocGenerics

    ## Loading required package: generics

    ##
    ## Attaching package: 'generics'

    ## The following objects are masked from 'package:base':
    ##
    ##     as.difftime, as.factor, as.ordered, intersect, is.element, setdiff,
    ##     setequal, union

    ##
    ## Attaching package: 'BiocGenerics'

    ## The following objects are masked from 'package:stats':
    ##
    ##     IQR, mad, sd, var, xtabs

    ## The following objects are masked from 'package:base':
    ##
    ##     anyDuplicated, aperm, append, as.data.frame, basename, cbind,
    ##     colnames, dirname, do.call, duplicated, eval, evalq, Filter, Find,
    ##     get, grep, grepl, is.unsorted, lapply, Map, mapply, match, mget,
    ##     order, paste, pmax, pmax.int, pmin, pmin.int, Position, rank,
    ##     rbind, Reduce, rownames, sapply, saveRDS, table, tapply, unique,
    ##     unsplit, which.max, which.min

    ## Loading required package: S4Vectors

    ##
    ## Attaching package: 'S4Vectors'

    ## The following object is masked from 'package:utils':
    ##
    ##     findMatches

    ## The following objects are masked from 'package:base':
    ##
    ##     expand.grid, I, unname

    ## Loading required package: IRanges

    ## Loading required package: Seqinfo

    gr <- GRanges(sample(LETTERS[1:3],200,replace=TRUE), IRanges(seq_len(200), width=2))
    gr$propertyA <- factor(sample(letters[1:5],200,replace=TRUE))
    gr

    ## GRanges object with 200 ranges and 1 metadata column:
    ##         seqnames    ranges strand | propertyA
    ##            <Rle> <IRanges>  <Rle> |  <factor>
    ##     [1]        A       1-2      * |         b
    ##     [2]        C       2-3      * |         b
    ##     [3]        B       3-4      * |         c
    ##     [4]        B       4-5      * |         a
    ##     [5]        A       5-6      * |         b
    ##     ...      ...       ...    ... .       ...
    ##   [196]        C   196-197      * |         a
    ##   [197]        C   197-198      * |         d
    ##   [198]        B   198-199      * |         b
    ##   [199]        A   199-200      * |         e
    ##   [200]        C   200-201      * |         a
    ##   -------
    ##   seqinfo: 3 sequences from an unspecified genome; no seqlengths

Again the file can then be loaded (without having all the data in memory) in the following way:

    f2 <- file.path(tmp, "test2")
    saveIndexedFst(gr, index.by="seqnames", file.prefix=f2)
    d1 <- loadIndexedFst(f2)
    names(d1)

    ## [1] "A" "C" "B"

    head(d1$A)

    ## GRanges object with 6 ranges and 1 metadata column:
    ##       seqnames    ranges strand | propertyA
    ##          <Rle> <IRanges>  <Rle> |  <factor>
    ##   [1]        A       1-2      * |         b
    ##   [2]        A       5-6      * |         b
    ##   [3]        A     14-15      * |         c
    ##   [4]        A     18-19      * |         c
    ##   [5]        A     22-23      * |         d
    ##   [6]        A     25-26      * |         b
    ##   -------
    ##   seqinfo: 3 sequences from an unspecified genome; no seqlengths

Similarly, we could index using a different column:

    saveIndexedFst(gr, index.by="propertyA", file.prefix=f2)
    d2 <- loadIndexedFst(f2)
    names(d2)

    ## [1] "a" "b" "c" "d" "e"

# More…

## Multithreading

The fst package supports multithreaded reading and writing. This can also be applied for `IndexedFst`, using the `nthreads` argument of `loadIndexedFst` and `saveIndexedFst`.

## Under the hood

The `IndexedFst` class is simply a wrapper around the `fst` package. In addition to the `fst` file, an `rds` file is saved containing the index data. For example, for our last example, the following files have been saved:

    list.files(tmp, "test2")

    ## [1] "test2.fst"     "test2.idx.rds"

Either file (or the prefix) can be used for loading, but both files need to have the same prefix.

# Session info

    ## R version 4.6.0 (2026-04-24)
    ## Platform: x86_64-pc-linux-gnu
    ## Running under: Ubuntu 24.04.4 LTS
    ##
    ## Matrix products: default
    ## BLAS:   /usr/lib/x86_64-linux-gnu/openblas-pthread/libblas.so.3
    ## LAPACK: /usr/lib/x86_64-linux-gnu/openblas-pthread/libopenblasp-r0.3.26.so;  LAPACK version 3.12.0
    ##
    ## locale:
    ##  [1] LC_CTYPE=en_US.UTF-8       LC_NUMERIC=C
    ##  [3] LC_TIME=en_US.UTF-8        LC_COLLATE=en_US.UTF-8
    ##  [5] LC_MONETARY=en_US.UTF-8    LC_MESSAGES=en_US.UTF-8
    ##  [7] LC_PAPER=en_US.UTF-8       LC_NAME=C
    ##  [9] LC_ADDRESS=C               LC_TELEPHONE=C
    ## [11] LC_MEASUREMENT=en_US.UTF-8 LC_IDENTIFICATION=C
    ##
    ## time zone: Etc/UTC
    ## tzcode source: system (glibc)
    ##
    ## attached base packages:
    ## [1] stats4    stats     graphics  grDevices utils     datasets  methods
    ## [8] base
    ##
    ## other attached packages:
    ##  [1] GenomicRanges_1.64.0 Seqinfo_1.2.0        IRanges_2.46.0
    ##  [4] S4Vectors_0.50.0     BiocGenerics_0.58.0  generics_0.1.4
    ##  [7] fstcore_0.10.0       scanMiRApp_1.18.0    scanMiR_1.16.0
    ## [10] BiocStyle_2.40.0
    ##
    ## loaded via a namespace (and not attached):
    ##   [1] RColorBrewer_1.1-3          sys_3.4.3
    ##   [3] jsonlite_2.0.0              magrittr_2.0.5
    ##   [5] shinyjqui_0.4.1             GenomicFeatures_1.64.0
    ##   [7] farver_2.1.2                rmarkdown_2.31
    ##   [9] BiocIO_1.22.0               vctrs_0.7.3
    ##  [11] memoise_2.0.1               Rsamtools_2.28.0
    ##  [13] RCurl_1.98-1.18             htmltools_0.5.9
    ##  [15] S4Arrays_1.12.0             progress_1.2.3
    ##  [17] AnnotationHub_4.2.0         curl_7.1.0
    ##  [19] SparseArray_1.12.0          sass_0.4.10
    ##  [21] bslib_0.10.0                htmlwidgets_1.6.4
    ##  [23] httr2_1.2.2                 plotly_4.12.0
    ##  [25] cachem_1.1.0                buildtools_1.0.0
    ##  [27] GenomicAlignments_1.48.0    mime_0.13
    ##  [29] lifecycle_1.0.5             pkgconfig_2.0.3
    ##  [31] Matrix_1.7-5                R6_2.6.1
    ##  [33] fastmap_1.2.0               MatrixGenerics_1.24.0
    ##  [35] shiny_1.13.0                digest_0.6.39
    ##  [37] AnnotationDbi_1.74.0        shinycssloaders_1.1.0
    ##  [39] RSQLite_2.4.6               seqLogo_1.78.0
    ##  [41] filelock_1.0.3              httr_1.4.8
    ##  [43] abind_1.4-8                 compiler_4.6.0
    ##  [45] bit64_4.8.0                 S7_0.2.2
    ##  [47] BiocParallel_1.46.0         DBI_1.3.0
    ##  [49] biomaRt_2.68.0              rappdirs_0.3.4
    ##  [51] DelayedArray_0.38.1         waiter_0.2.5.1
    ##  [53] rjson_0.2.23                tools_4.6.0
    ##  [55] otel_0.2.0                  httpuv_1.6.17
    ##  [57] fst_0.9.8                   glue_1.8.1
    ##  [59] restfulr_0.0.16             promises_1.5.0
    ##  [61] grid_4.6.0                  gtable_0.3.6
    ##  [63] tidyr_1.3.2                 ensembldb_2.36.0
    ##  [65] data.table_1.18.2.1         hms_1.1.4
    ##  [67] XVector_0.52.0              stringr_1.6.0
    ##  [69] BiocVersion_3.23.1          pillar_1.11.1
    ##  [71] later_1.4.8                 rintrojs_0.3.4
    ##  [73] dplyr_1.2.1                 BiocFileCache_3.2.0
    ##  [75] lattice_0.22-9              rtracklayer_1.72.0
    ##  [77] bit_4.6.0                   tidyselect_1.2.1
    ##  [79] maketools_1.3.2             Biostrings_2.80.0
    ##  [81] knitr_1.51                  ProtGenerics_1.44.0
    ##  [83] SummarizedExperiment_1.42.0 xfun_0.57
    ##  [85] shinydashboard_0.7.3        Biobase_2.72.0
    ##  [87] matrixStats_1.5.0           DT_0.34.0
    ##  [89] stringi_1.8.7               UCSC.utils_1.8.0
    ##  [91] lazyeval_0.2.3              yaml_2.3.12
    ##  [93] evaluate_1.0.5              codetools_0.2-20
    ##  [95] cigarillo_1.2.0             tibble_3.3.1
    ##  [97] BiocManager_1.30.27         cli_3.6.6
    ##  [99] xtable_1.8-8                jquerylib_0.1.4
    ## [101] Rcpp_1.1.1-1.1              GenomeInfoDb_1.48.0
    ## [103] dbplyr_2.5.2                png_0.1-9
    ## [105] XML_3.99-0.23               parallel_4.6.0
    ## [107] ggplot2_4.0.3               blob_1.3.0
    ## [109] prettyunits_1.2.0           AnnotationFilter_1.36.0
    ## [111] bitops_1.0-9                pwalign_1.8.0
    ## [113] txdbmaker_1.8.0             viridisLite_0.4.3
    ## [115] scales_1.4.0                scanMiRData_1.16.0
    ## [117] purrr_1.2.2                 crayon_1.5.3
    ## [119] rlang_1.2.0                 cowplot_1.2.0
    ## [121] KEGGREST_1.52.0