AscendIndexIVFSQ with PCAR

The large library algorithm AscendIndexIVFSQ can be trained through quantization and IVF based on a group of data to generate appropriate quantization and bucket division functions. For the input float32 feature vectors, AscendIndexIVFSQ quantizes them into int8 feature vectors and allocates them to specified buckets. During vector comparison, AscendIndexIVFSQ selects nprobe buckets that are closest to the to-be-compared vector and dequantizes int8 feature vectors to the original feature vectors for subsequent calculation. PCAR is the default dimension reduction algorithm provided by mxIndex and can further reduce the dimension of high-dimensional vectors by analyzing the principal components, so as to achieve higher retrieval speed and storage capacity. In addition, mxIndex also supports user-defined neural network dimension reduction. The following is a typical sample of AscendIndexIVFSQ with PCAR:

#include <faiss/ascend/AscendIndexIVFSQ.h>
#include <faiss/ascend/AscendIndexPreTransform.h>
#include <iostream>
#include <random>

using namespace std;

int main(int argc, char **argv)
{
    const size_t dim = 128;
    const size_t dimReduced = 64;
    const size_t ntotal = 10000;
    const size_t ncentroids = 1024;
    vector<float> data(dim * ntotal);
    for (int i = 0; i < data.size(); i++) {
        data[i] = drand48();
    }

    const size_t k = 100;
    const size_t searchNum = 100;
    const size_t nprobe = 24;

    vector<float> dist(k * searchNum);
    vector<long> indices(k * searchNum);

    faiss::ascend::AscendIndexPreTransform *index = nullptr;
    try {
        faiss::ascend::AscendIndexIVFSQConfig conf({0}, 768);
        faiss::ascend::AscendIndexIVFSQ *subIndex = new faiss::ascend::AscendIndexIVFSQ(
            dimReduced, ncentroids, faiss::ScalarQuantizer::QuantizerType::QT_8bit, faiss::METRIC_INNER_PRODUCT, false, conf);
        subIndex->verbose = true;
        subIndex->setNumProbes(nprobe);

        index = new faiss::ascend::AscendIndexPreTransform(subIndex);
        index->verbose = true;
        index->prependTransform<faiss::ascend::AscendPCAMatrix>(dim, dimReduced, 0.0f, true);
        index->train(ntotal, data.data());
        index->add(ntotal, data.data());
        index->search(searchNum, data.data(), k, dist.data(), indices.data());
    } catch (...) {
        cout << "Exception caught!" << endl;
        delete index;
        return -1;
    }
    cout << "Search finished successfully" << endl;
    delete index;
    return 0;
}