/*
 * (C) Copyright 1996- ECMWF.
 *
 * This software is licensed under the terms of the Apache Licence Version 2.0
 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
 * In applying this licence, ECMWF does not waive the privileges and immunities
 * granted to it by virtue of its status as an intergovernmental organisation nor
 * does it submit to any jurisdiction.
 */

/// @author Tiago Quintino
/// @date Sep 2012

#include <cstring>
#include <memory>

#include <dirent.h>
#include <fcntl.h>

#include "eckit/io/DataHandle.h"
#include "eckit/io/MemoryHandle.h"
#include "eckit/runtime/Main.h"
#include "eckit/types/Types.h"
#include "eckit/utils/Translator.h"
#include "eckit/serialisation/MemoryStream.h"
#include "eckit/serialisation/FileStream.h"
#include "eckit/io/AutoCloser.h"

#include "metkit/mars/MarsExpension.h"
#include "metkit/mars/MarsRequest.h"
#include "metkit/mars/TypeAny.h"

#include "fdb5/database/Key.h"
#include "fdb5/database/Archiver.h"
#include "fdb5/database/ArchiveVisitor.h"
#include "fdb5/api/FDB.h"
#include "fdb5/api/helpers/FDBToolRequest.h"

#include "eckit/testing/Test.h"

using namespace std;
using namespace eckit;
using namespace eckit::testing;
using namespace fdb5;


namespace fdb {
namespace test {

//----------------------------------------------------------------------------------------------------------------------

struct FixtureService {

    metkit::mars::MarsRequest env;
    StringDict p;
    fdb5::Config config;

    std::vector<std::string> modelParams_;

    FixtureService() : env("environ")
    {
        p["class"]  = "rd";
        p["stream"] = "oper";
        p["domain"] = "g";
        p["expver"] = "0001";
        p["date"] = "20120911";
        p["time"] = "0000";
        p["type"] = "fc";

        modelParams_.push_back( "130.128" );
        modelParams_.push_back( "138.128" );
    }

    void write_cycle(fdb5::Archiver& fdb, StringDict& p )
    {
        Translator<size_t,std::string> str;
        std::vector<std::string>::iterator param = modelParams_.begin();
        for( ; param != modelParams_.end(); ++param )
        {
            p["param"] = *param;

            p["levtype"] = "pl";

            for( size_t step = 0; step < 12; ++step )
            {
                p["step"] = str(step*3);

                for( size_t level = 0; level < 10; ++level )
                {
                    p["levelist"] = str(level*100);

                    std::ostringstream data;
                    data << "Raining cats and dogs -- "
                         << " param " << *param
                         << " step "  << step
                         << " level " << level
                         << std::endl;
                    std::string data_str = data.str();

                    fdb5::Key k{p};
                    ArchiveVisitor visitor(fdb, k, static_cast<const void *>(data_str.c_str()), data_str.size());
                    fdb.archive(k, visitor);
                }
            }
        }
    }
};

//----------------------------------------------------------------------------------------------------------------------

CASE ( "test_fdb_stepunit_archive" ) {

    std::string data_str = "Raining cats and dogs";

    fdb5::FDB fdb;

    Key key;
    key.set("class","od");
    key.set("expver","0001");
    key.set("type","fc");
    key.set("stream","oper");
    key.set("date","20101010");
    key.set("time","0000");
    key.set("domain","g");
    key.set("levtype","sfc");
    key.set("step","60");
    key.set("param","130");
    fdb.archive(key, static_cast<const void *>(data_str.c_str()), data_str.size());
    fdb.flush();

    metkit::mars::MarsRequest req = key.request();

    {
        fdb5::FDBToolRequest r(req);
        fdb5::ListIterator iter = fdb.list(r, true);
        fdb5::ListElement el;
        EXPECT(iter.next(el));
        EXPECT(el.combinedKey().get("step") == "60");
        EXPECT(!iter.next(el));
    }

    key.set("step","2");
    key.set("stepunits","h");
    fdb.archive(key, static_cast<const void *>(data_str.c_str()), data_str.size());
    fdb.flush();

    req.setValue("step", "2");
    {
        fdb5::FDBToolRequest r(req);
        fdb5::ListIterator iter = fdb.list(r, true);
        fdb5::ListElement el;
        EXPECT(iter.next(el));
        EXPECT(el.combinedKey().get("step") == "2");
        EXPECT(!iter.next(el));
    }

    req.setValuesTyped(new metkit::mars::TypeAny("step"), std::vector<std::string>{"2","60"});
    {
        fdb5::FDBToolRequest r(req);
        fdb5::ListIterator iter = fdb.list(r, true);
        fdb5::ListElement el;
        EXPECT(iter.next(el));
        EXPECT(el.combinedKey().get("step") == "2");
        EXPECT(iter.next(el));
        EXPECT(el.combinedKey().get("step") == "60");
        EXPECT(!iter.next(el));
    }

    key.set("step","30");
    key.set("stepunits","m");
    fdb.archive(key, static_cast<const void *>(data_str.c_str()), data_str.size());
    fdb.flush();

    req.setValue("step", "30m");
    {
        fdb5::FDBToolRequest r(req);
        fdb5::ListIterator iter = fdb.list(r, true);
        fdb5::ListElement el;
        EXPECT(iter.next(el));
        EXPECT(el.combinedKey().get("step") == "30m");
        EXPECT(!iter.next(el));
    }

    req.values("step", {"0","to","2","by","30m"});
    req.unsetValues("param");
    {
        metkit::mars::MarsExpension expand{false};

        metkit::mars::MarsRequest expandedRequests = expand.expand(req);
        fdb5::FDBToolRequest r(expandedRequests);
        fdb5::ListIterator iter = fdb.list(r, true);
        fdb5::ListElement el;
        EXPECT(iter.next(el));
        EXPECT(el.combinedKey().get("step") == "30m");
        EXPECT(iter.next(el));
        EXPECT(el.combinedKey().get("step") == "2");
        EXPECT(!iter.next(el));
    }
}

CASE ( "test_fdb_service" ) {

    SETUP("Fixture") {
        FixtureService f;

        SECTION( "test_fdb_service_write" )
        {
            fdb5::Archiver fdb;

            f.p["class"]  = "rd";
            f.p["stream"] = "oper";
            f.p["domain"] = "g";
            f.p["expver"] = "0001";

            f.p["date"] = "20120911";
            f.p["time"] = "0000";
            f.p["type"] = "fc";

            f.write_cycle(fdb, f.p);

            f.p["date"] = "20120911";
            f.p["time"] = "0000";
            f.p["type"] = "4v";

            f.write_cycle(fdb, f.p);

            f.p["date"] = "20120912";
            f.p["time"] = "0000";
            f.p["type"] = "fc";

            f.write_cycle(fdb, f.p);

            f.p["date"] = "20120912";
            f.p["time"] = "0000";
            f.p["type"] = "4v";

            f.write_cycle(fdb, f.p);
        }


        SECTION( "test_fdb_service_readtobuffer" )
        {
            fdb5::FDB retriever;

            Buffer buffer(1024);

            Translator<size_t,std::string> str;
            std::vector<std::string>::iterator param = f.modelParams_.begin();
            for( ; param != f.modelParams_.end(); ++param )
            {
                f.p["param"] = *param;
                f.p["levtype"] = "pl";

                for( size_t step = 0; step < 2; ++step )
                {
                    f.p["step"] = str(step*3);

                    for( size_t level = 0; level < 3; ++level )
                    {
                        f.p["levelist"] = str(level*100);

                        Log::info() << "Looking for: " << f.p << std::endl;

                        metkit::mars::MarsRequest r("retrieve", f.p);
                        std::unique_ptr<DataHandle> dh ( retriever.retrieve(r) );  AutoClose closer1(*dh);

                        ::memset(buffer, 0, buffer.size());

                        dh->openForRead();
                        dh->read(buffer, buffer.size());

                        Log::info() << (char*) buffer << std::endl;

                        std::ostringstream data;
                        data << "Raining cats and dogs -- "
                                << " param " << *param
                                << " step "  << step
                                << " level " << level
                                << std::endl;

                        EXPECT(::memcmp(buffer, data.str().c_str(), data.str().size()) == 0);
                    }
                }
            }
        }


        SECTION( "test_fdb_service_list" )
        {
            fdb5::FDB lister;

            Translator<size_t,std::string> str;
            std::vector<std::string>::iterator param = f.modelParams_.begin();
            for( ; param != f.modelParams_.end(); ++param )
            {
                f.p["param"] = *param;
                f.p["levtype"] = "pl";
                f.p["step"] = str(0);
                f.p["levelist"] = str(0);

                Log::info() << "Looking for: " << f.p << std::endl;

                metkit::mars::MarsRequest r("retrieve", f.p);
                fdb5::FDBToolRequest req(r);
                auto iter = lister.list(req);
                fdb5::ListElement el;
                EXPECT(iter.next(el));

                eckit::PathName path = el.uri().path().dirName();

                DIR* dirp = ::opendir(path.asString().c_str());
                struct dirent* dp;
                while ((dp = ::readdir(dirp)) != nullptr) {
                    EXPECT_NOT(strstr( dp->d_name, "toc."));
                }
                ::closedir(dirp);

                // consuming the rest of the queue
                while (iter.next(el));
            }
        }

        SECTION( "test_fdb_service_marsreques" )
        {
            std::vector<string> steps;
            steps.push_back( "15" );
            steps.push_back( "18" );
            steps.push_back( "24" );

            std::vector<string> levels;
            levels.push_back( "100" );
            levels.push_back( "500" );

            std::vector<string> params;
            params.push_back( "130.128" );
            params.push_back( "138.128" );

            std::vector<string> dates;
            dates.push_back( "20120911" );
            dates.push_back( "20120912" );


            metkit::mars::MarsRequest r("retrieve");

            r.setValue("class","rd");
            r.setValue("expver","0001");
            r.setValue("type","fc");
            r.setValue("stream","oper");
            r.setValue("time","0000");
            r.setValue("domain","g");
            r.setValue("levtype","pl");

            r.setValuesTyped(new metkit::mars::TypeAny("param"), params);
            r.setValuesTyped(new metkit::mars::TypeAny("step"), steps);
            r.setValuesTyped(new metkit::mars::TypeAny("levelist"), levels);
            r.setValuesTyped(new metkit::mars::TypeAny("date"), dates);

            Log::info() << r << std::endl;

            fdb5::FDB retriever;

            std::unique_ptr<DataHandle> dh ( retriever.retrieve(r) );

            PathName path ( "data_mars_request.data" );
            path.unlink();

            dh->saveInto(path);
        }
    }
}



CASE ( "test_fdb_service_subtoc" ) {

    SETUP("Fixture") {
        FixtureService f;

        eckit::LocalConfiguration userConf;
        userConf.set("useSubToc", true);

        fdb5::Config expanded = fdb5::Config().expandConfig();

        fdb5::Config config(expanded, userConf);

        SECTION( "test_fdb_service_subtoc_write" )
        {
            fdb5::Archiver fdb(config);

            f.p["class"]  = "rd";
            f.p["stream"] = "oper";
            f.p["domain"] = "g";
            f.p["expver"] = "0002";

            f.p["date"] = "20120911";
            f.p["time"] = "0000";
            f.p["type"] = "fc";

            f.write_cycle(fdb, f.p);

            f.p["date"] = "20120911";
            f.p["time"] = "0000";
            f.p["type"] = "4v";

            f.write_cycle(fdb, f.p);

            f.p["date"] = "20120912";
            f.p["time"] = "0000";
            f.p["type"] = "fc";

            f.write_cycle(fdb, f.p);

            f.p["date"] = "20120912";
            f.p["time"] = "0000";
            f.p["type"] = "4v";

            f.write_cycle(fdb, f.p);
        }

        SECTION( "test_fdb_service_subtoc_readtobuffer" )
        {
            fdb5::FDB retriever;

            Buffer buffer(1024);

            f.p["expver"] = "0002";

            Translator<size_t,std::string> str;
            std::vector<std::string>::iterator param = f.modelParams_.begin();
            for( ; param != f.modelParams_.end(); ++param )
            {
                f.p["param"] = *param;
                f.p["levtype"] = "pl";

                for( size_t step = 0; step < 2; ++step )
                {
                    f.p["step"] = str(step*3);

                    for( size_t level = 0; level < 3; ++level )
                    {
                        f.p["levelist"] = str(level*100);

                        Log::info() << "Looking for: " << f.p << std::endl;

                        metkit::mars::MarsRequest r("retrieve", f.p);
                        std::unique_ptr<DataHandle> dh ( retriever.retrieve(r) );  AutoClose closer1(*dh);

                        ::memset(buffer, 0, buffer.size());

                        dh->openForRead();
                        dh->read(buffer, buffer.size());

                        Log::info() << (char*) buffer << std::endl;

                        std::ostringstream data;
                        data << "Raining cats and dogs -- "
                                << " param " << *param
                                << " step "  << step
                                << " level " << level
                                << std::endl;

                        EXPECT(::memcmp(buffer, data.str().c_str(), data.str().size()) == 0);
                    }
                }
            }
        }


        SECTION( "test_fdb_service_subtoc_list" )
        {
            fdb5::FDB lister;

            f.p["expver"] = "0002";

            Translator<size_t,std::string> str;
            std::vector<std::string>::iterator param = f.modelParams_.begin();
            for( ; param != f.modelParams_.end(); ++param )
            {

                f.p["param"] = *param;
                f.p["levtype"] = "pl";
                f.p["step"] = str(0);
                f.p["levelist"] = str(0);

                Log::info() << "Looking for: " << f.p << std::endl;

                metkit::mars::MarsRequest r("retrieve", f.p);
                fdb5::FDBToolRequest req(r);
                auto iter = lister.list(req);
                fdb5::ListElement el;
                EXPECT(iter.next(el));

                eckit::PathName path = el.uri().path().dirName();

                DIR* dirp = ::opendir(path.asString().c_str());
                struct dirent* dp;
                bool subtoc = false;
                while ((dp = ::readdir(dirp)) != nullptr) {
                    if (strstr( dp->d_name, "toc.")) {
                        subtoc = true;
                    }
                }
                EXPECT(subtoc);
                ::closedir(dirp);

                // consuming the rest of the queue
                while (iter.next(el));
            }
        }

        SECTION( "test_fdb_service_subtoc_marsreques" )
        {
            std::vector<string> steps;
            steps.push_back( "15" );
            steps.push_back( "18" );
            steps.push_back( "24" );

            std::vector<string> levels;
            levels.push_back( "100" );
            levels.push_back( "500" );

            std::vector<string> params;
            params.push_back( "130.128" );
            params.push_back( "138.128" );

            std::vector<string> dates;
            dates.push_back( "20120911" );
            dates.push_back( "20120912" );


            metkit::mars::MarsRequest r("retrieve");

            r.setValue("class","rd");
            r.setValue("expver","0002");
            r.setValue("type","fc");
            r.setValue("stream","oper");
            r.setValue("time","0000");
            r.setValue("domain","g");
            r.setValue("levtype","pl");

            r.setValuesTyped(new metkit::mars::TypeAny("param"), params);
            r.setValuesTyped(new metkit::mars::TypeAny("step"), steps);
            r.setValuesTyped(new metkit::mars::TypeAny("levelist"), levels);
            r.setValuesTyped(new metkit::mars::TypeAny("date"), dates);

            Log::info() << r << std::endl;

            fdb5::FDB retriever;

            std::unique_ptr<DataHandle> dh ( retriever.retrieve(r) );

            PathName path ( "data_mars_request.data" );
            path.unlink();

            dh->saveInto(path);
        }
    }
}

CASE( "schemaSerialisation" ) {

    PathName filename    = PathName::unique("data");
    std::string filepath = filename.asString();

    std::string original;

    {
        eckit::FileStream sout(filepath.c_str(), "w");
        auto c = eckit::closer(sout);

        fdb5::Config config;
        config = config.expandConfig();

        std::stringstream ss;
        config.schema().dump(ss);
        original = ss.str();

        sout << config.schema();
    }
    {
        eckit::FileStream sin(filepath.c_str(), "r");
        auto c = eckit::closer(sin);

        std::unique_ptr<fdb5::Schema> clone(eckit::Reanimator<fdb5::Schema>::reanimate(sin));

        std::stringstream ss;
        clone->dump(ss);

        EXPECT(original == ss.str());
    }
    if (filename.exists()) {
        filename.unlink();
    }
}

//-----------------------------------------------------------------------------

}  // namespace test
}  // namespace fdb

int main(int argc, char **argv)
{
    eckit::Main::initialise(argc, argv, "FDB_HOME");
    return run_tests ( argc, argv, false );
}
