#include <stdio.h>
#include <stdlib.h>
#include <assert.h>
#include <err.h>

#include <gsl/gsl_rng.h>
#include <tskit/tables.h>

#define check_tsk_error(val) if (val < 0) {\
    errx(EXIT_FAILURE, "line %d: %s", __LINE__, tsk_strerror(val));\
}

void
simulate(tsk_table_collection_t *tables, int N, int T, int simplify_interval, gsl_rng *rng)
{
    tsk_id_t *buffer, *parents, *children, child, left_parent, right_parent;
    double breakpoint;
    int ret, j, t, b;

    assert(simplify_interval != 0); // leads to division by zero
    buffer = malloc(2 * N * sizeof(tsk_id_t));
    if (buffer == NULL) {
        errx(EXIT_FAILURE, "Out of memory");
    }
    tables->sequence_length = 1.0;
    parents = buffer;
    for (j = 0; j < N; j++) {
        parents[j] = tsk_node_table_add_row(&tables->nodes, 0, T,
                TSK_NULL, TSK_NULL, NULL, 0);
        check_tsk_error(parents[j]);
    }
    b = 0;
    for (t = T - 1; t >= 0; t--) {
        /* Alternate between using the first and last N values in the buffer */
        parents = buffer + (b * N);
        b = (b + 1) % 2;
        children = buffer + (b * N);
        for (j = 0; j < N; j++) {
            child = tsk_node_table_add_row(&tables->nodes, 0, t,
                    TSK_NULL, TSK_NULL, NULL, 0);
            check_tsk_error(child);
            left_parent = parents[gsl_rng_uniform_int(rng, N)];
            right_parent = parents[gsl_rng_uniform_int(rng, N)];
            do {
                breakpoint = gsl_rng_uniform(rng);
            } while (breakpoint == 0); /* tiny proba of breakpoint being 0 */
            ret = tsk_edge_table_add_row(&tables->edges, 0, breakpoint, left_parent, child);
            check_tsk_error(ret);
            ret = tsk_edge_table_add_row(&tables->edges, breakpoint, 1, right_parent, child);
            check_tsk_error(ret);
            children[j] = child;
        }
        if (t % simplify_interval == 0) {
            printf("Simplify at generation %d: (%d nodes %d edges)", t,
                    tables->nodes.num_rows, tables->edges.num_rows);
            /* Note: Edges must be sorted for simplify to work, and we use a brute force
             * approach of sorting each time here for simplicity. This is inefficient. */
            ret = tsk_table_collection_sort(tables, NULL, 0);
            check_tsk_error(ret);
            ret = tsk_table_collection_simplify(tables, children, N, 0, NULL);
            check_tsk_error(ret);
            printf(" -> (%d nodes %d edges)\n", tables->nodes.num_rows,
                    tables->edges.num_rows);
            for (j = 0; j < N; j++) {
                children[j] = j;
            }
        }
    }
    free(buffer);
}

int
main(int argc, char **argv)
{
    int ret;
    tsk_table_collection_t tables;
    gsl_rng *rng = gsl_rng_alloc(gsl_rng_default);

    if (argc != 5) {
        errx(EXIT_FAILURE, "usage: N T simplify-interval output-file");
    }
    ret = tsk_table_collection_init(&tables, 0);
    check_tsk_error(ret);
    simulate(&tables, atoi(argv[1]), atoi(argv[2]), atoi(argv[3]), rng);
    ret = tsk_table_collection_dump(&tables, argv[4], 0);
    check_tsk_error(ret);

    tsk_table_collection_free(&tables);
    gsl_rng_free(rng);
    return 0;
}


/* This file is auto-generated from countpairs_s_mu_mocks_impl.c.src */
#ifndef DOUBLE_PREC
#define DOUBLE_PREC
#endif
// # -*- mode: c -*-
/* File: countpairs_s_mu_mocks_impl.c.src */
/*
  This file is a part of the Corrfunc package
  Copyright (C) 2015-- Manodeep Sinha (manodeep@gmail.com)
  License: MIT LICENSE. See LICENSE file under the top-level
  directory at https://github.com/manodeep/Corrfunc/
*/

#include <stdio.h>
#include <math.h>
#include <stdlib.h>
#include <signal.h>
#include <unistd.h>
#include <gsl/gsl_interp.h>

#include "countpairs_s_mu_mocks_impl_double.h"
#include "countpairs_s_mu_mocks_kernels_double.c"
#include "cellarray_mocks_double.h"
#include "gridlink_mocks_impl_double.h"

#include "defs.h"
#include "utils.h"
#include "cosmology_params.h"
#include "set_cosmo_dist.h"
#include "cpu_features.h"
#include "progressbar.h"
#include "proj_functions_double.h"

#if defined(_OPENMP)
#include <omp.h>
#endif

int interrupt_status_DDsmu_mocks_double=EXIT_SUCCESS;

void interrupt_handler_countpairs_s_mu_mocks_double(int signo)
{
    fprintf(stderr,"Received signal = `%s' (signo = %d). Aborting \n",strsignal(signo), signo);
    interrupt_status_DDsmu_mocks_double = EXIT_FAILURE;
}

int check_ra_dec_cz_s_mu_double(const int64_t N, double *phi, double *theta, double *cz)
{

    if(N==0) {
        return EXIT_SUCCESS;
    }
    if(phi == NULL || theta == NULL || cz == NULL) {
        fprintf(stderr,"Input arrays can not be NULL. Have RA = %p DEC = %p cz = %p\n", phi, theta, cz);
        return EXIT_FAILURE;
    }

    int fix_cz  = 0;
    int fix_ra  = 0;
    int fix_dec = 0;

    const double max_cz_threshold = 10.0;//if I find that max cz is smaller than this threshold, then I will assume z has been supplied rather than cz
    double max_cz = 0.0;
    //Check input cz -> ensure that cz contains cz and not z
    for(int64_t i=0;i<N;i++) {
        if(cz[i] > max_cz) max_cz = cz[i];
        if(phi[i] < 0.0) {
            fix_ra = 1;
        }
        if(theta[i] > 90.0) {
            fix_dec = 1;
        }
        if(theta[i] > 180) {
            fprintf(stderr,"theta[%"PRId64"] = %"REAL_FORMAT"should be less than 180 deg\n", i, theta[i]);
            return EXIT_FAILURE;
        }
    }
    if(max_cz < max_cz_threshold) fix_cz = 1;

    //Only run the loop if something needs to be fixed
    if(fix_cz==1 || fix_ra == 1 || fix_dec == 1) {
        if(fix_ra == 1) {
            fprintf(stderr,"%s> Out of range values found for ra. Expected ra to be in the range [0.0,360.0]. Found ra values in [-180,180] -- fixing that\n", __FUNCTION__);
        }
        if(fix_dec == 1) {
            fprintf(stderr,"%s> Out of range values found for dec. Expected dec to be in the range [-90.0,90.0]. Found dec values in [0,180] -- fixing that\n", __FUNCTION__);
        }
        if(fix_cz == 1)  {
            fprintf(stderr,"%s> Out of range values found for cz. Expected input to be `cz' but found `z' instead. max_cz (found in input) = %"REAL_FORMAT" threshold "
                    "= %"REAL_FORMAT"\n",__FUNCTION__,max_cz,max_cz_threshold);
        }

        for(int64_t i=0;i<N;i++) {
            if(fix_ra==1) {
                phi[i] += (double) 180.0;
            }
            if(fix_dec==1) {
                theta[i] -= (double) 90.0;
            }
            if(fix_cz == 1) {
                cz[i] *= (double) SPEED_OF_LIGHT;//input was z -> convert to cz
            }
        }
    }

    return EXIT_SUCCESS;
}

countpairs_mocks_func_ptr_double countpairs_s_mu_mocks_driver_double(const struct config_options *options)
{

    static countpairs_mocks_func_ptr_double function = NULL;
    static isa old_isa=-1;
    if(old_isa == options->instruction_set) {
        return function;
    }

    /* Array of function pointers */
    countpairs_mocks_func_ptr_double allfunctions[] = {
#ifdef __AVX__
        countpairs_s_mu_mocks_avx_intrinsics_double,
#endif
#ifdef __SSE4_2__
      countpairs_s_mu_mocks_sse_intrinsics_double,
#endif
      countpairs_s_mu_mocks_fallback_double
    };

    const int num_functions = sizeof(allfunctions)/sizeof(void *);
    const int fallback_offset = num_functions - 1;
#if defined(__AVX__) || defined __SSE4_2__
    const int highest_isa = instrset_detect();
#endif
    int curr_offset = 0;

    /* Now check if AVX is supported by the CPU */
    int avx_offset = fallback_offset;
#ifdef __AVX__
    avx_offset = highest_isa >= 7 ? curr_offset:fallback_offset;
    curr_offset++;
#endif

    /* Is the SSE function supported at runtime and enabled at compile-time?*/
    int sse_offset = fallback_offset;
#ifdef __SSE4_2__
    sse_offset = highest_isa >= 6 ? curr_offset:fallback_offset;
    curr_offset++;
#endif
    if( curr_offset != fallback_offset) {
      fprintf(stderr,"ERROR: Bug in code (current offset = %d *should equal* fallback function offset = %d)\n",
              curr_offset, fallback_offset);
      return NULL;
    }

    int function_dispatch=0;
    /* Check that cpu supports feature */
    if(options->instruction_set >= 0) {
        switch(options->instruction_set) {
        case(AVX512F):
        case(AVX2):
        case(AVX):function_dispatch=avx_offset;break;
        case(SSE42): function_dispatch=sse_offset;break;
        default:function_dispatch=fallback_offset;break;
        }
    }

    if(function_dispatch >= num_functions) {
      fprintf(stderr,"In %s> ERROR: Could not resolve the correct function.\n Function index = %d must lie between [0, %d)\n",
              __FUNCTION__, function_dispatch, num_functions);
      return NULL;
    }
    function = allfunctions[function_dispatch];
    old_isa = options->instruction_set;

    if(options->verbose){
        // This must be first (AVX/SSE may be aliased to fallback)
        if(function_dispatch == fallback_offset){
            fprintf(stderr,"Using fallback kernel\n");
        } else if(function_dispatch == avx_offset){
            fprintf(stderr,"Using AVX kernel\n");
        } else if(function_dispatch == sse_offset){
            fprintf(stderr,"Using SSE kernel\n");
        } else {
            printf("Unknown kernel!\n");
        }
    }

    return function;
}

int countpairs_mocks_s_mu_double(const int64_t ND1, double *ra1, double *dec1, double *czD1,
                                 const int64_t ND2, double *ra2, double *dec2, double *czD2,
                                 const int numthreads,
                                 const int autocorr,
                                 const char *sbinfile,
                                 const double max_mu,
                                 const int nmu_bins,
                                 const int cosmology,
                                 results_countpairs_mocks_s_mu *results,
                                 struct config_options *options, struct extra_options *extra)
{

    if(options->float_type != sizeof(double)) {
        fprintf(stderr,"ERROR: In %s> Can only handle arrays of size=%zu. Got an array of size = %zu\n",
                __FUNCTION__, sizeof(double), options->float_type);
        return EXIT_FAILURE;
    }

    // If no extra options were passed, create dummy options
    // This allows us to pass arguments like "extra->weights0" below;
    // they'll just be NULLs, which is the correct behavior
    struct extra_options dummy_extra;
    if(extra == NULL){
      weight_method_t dummy_method = NONE;
      dummy_extra = get_extra_options(dummy_method);
      extra = &dummy_extra;
    }

    int need_weightavg = extra->weight_method != NONE;

    options->sort_on_z = 1;
    struct timeval t0;
    if(options->c_api_timer) {
        gettimeofday(&t0, NULL);
    }
    if(options->fast_divide_and_NR_steps >= MAX_FAST_DIVIDE_NR_STEPS) {
        fprintf(stderr, ANSI_COLOR_MAGENTA"Warning: The number of requested Newton-Raphson steps = %u is larger than max. allowed steps = %u."
                " Switching to a standard divide"ANSI_COLOR_RESET"\n",
                options->fast_divide_and_NR_steps, MAX_FAST_DIVIDE_NR_STEPS);
        options->fast_divide_and_NR_steps = 0;
    }

    //Check inputs
    if(ND1 == 0 || (autocorr == 0 && ND2 == 0)) {
        return EXIT_SUCCESS;
    }

    //Check inputs
    int status1 = check_ra_dec_cz_s_mu_double(ND1, ra1, dec1, czD1);
    if(status1 != EXIT_SUCCESS) {
        return status1;
    }
    if(autocorr==0) {
        int status2 = check_ra_dec_cz_s_mu_double(ND2, ra2, dec2, czD2);
        if(status2 != EXIT_SUCCESS) {
            return status2;
        }
    }

#if defined(_OPENMP)
    omp_set_num_threads(numthreads);
#else
    (void) numthreads;
#endif

    if(options->max_cells_per_dim == 0) {
        fprintf(stderr,"Warning: Max. cells per dimension is set to 0 - resetting to `NLATMAX' = %d\n", NLATMAX);
        options->max_cells_per_dim = NLATMAX;
    }
    for(int i=0;i<3;i++) {
        if(options->bin_refine_factors[i] < 1) {
            fprintf(stderr,"Warning: bin refine factor along axis = %d *must* be >=1. Instead found bin refine factor =%d\n",
                    i, options->bin_refine_factors[i]);
            reset_bin_refine_factors(options);
            break;/* all factors have been reset -> no point continuing with the loop */
        }
    }

    /* setup interrupt handler -> mostly useful during the python execution.
       Let's Ctrl-C abort the extension  */
    SETUP_INTERRUPT_HANDLERS(interrupt_handler_countpairs_s_mu_mocks_double);

    //Try to initialize cosmology - code will exit if comoslogy is not implemented.
    //Putting in a different scope so I can call the variable status
    {
        int status = init_cosmology(cosmology);
        if(status != EXIT_SUCCESS) {
            return status;
        }
    }

    /***********************
     *initializing the  bins
     ************************/
    double *supp;
    int nsbin;
    double smin,smax;
    setup_bins(sbinfile,&smin,&smax,&nsbin,&supp);
    if( ! (smin > 0.0 && smax > 0.0 && smin < smax && nsbin > 0)) {
        fprintf(stderr,"Error: Could not setup with S bins correctly. (smin = %lf, smax = %lf, with nbins = %d). Expected non-zero smin/smax with smax > smin and nbins >=1 \n",
                smin, smax, nsbin);
        return EXIT_FAILURE;
    }

    if(max_mu <= 0.0 || max_mu > 1.0) {
        fprintf(stderr,"Error: max_mu (max. value for the cosine of the angle with line of sight) must be greater than 0 and at most 1).\n"
                "The passed value is max_mu = %lf. Please change it to be > 0 and <= 1.0\n", max_mu);
        return EXIT_FAILURE;
    }

    if(nmu_bins < 1 ) {
        fprintf(stderr,"Error: Number of mu bins = %d must be at least 1\n", nmu_bins);
        return EXIT_FAILURE;
    }

    //Change cz into co-moving distance
    double *D1 = NULL, *D2 = NULL;
    if(options->is_comoving_dist == 0) {
        D1 = my_malloc(sizeof(*D1),ND1);
        D2 = autocorr == 0 ? my_malloc(sizeof(*D2),ND2):D1;
    } else {
        D1 = czD1;
        D2 = autocorr == 0 ? czD2:czD1;
    }

    if(D1 == NULL || D2 == NULL) {
        free(D1);free(D2);
        return EXIT_FAILURE;
    }

    if(options->is_comoving_dist == 0) {
        //Setup variables to do the cz->comoving distance
        double czmax = 0.0;
        const double inv_speed_of_light = 1.0/SPEED_OF_LIGHT;
        get_max_double(ND1, czD1, &czmax);
        if(autocorr == 0) {
            get_max_double(ND2, czD2, &czmax);
        }
        const double zmax = czmax * inv_speed_of_light + 0.01;

        const int workspace_size = 10000;
        double *interp_redshift  = my_calloc(sizeof(*interp_redshift), workspace_size);//the interpolation is done in 'z' and not in 'cz'
        double *interp_comoving_dist = my_calloc(sizeof(*interp_comoving_dist),workspace_size);
        int Nzdc = set_cosmo_dist(zmax, workspace_size, interp_redshift, interp_comoving_dist, cosmology);
        if(Nzdc < 0) {
            free(interp_redshift);free(interp_comoving_dist);
            return EXIT_FAILURE;
        }

        gsl_interp *interpolation;
        gsl_interp_accel *accelerator;
        accelerator =  gsl_interp_accel_alloc();
        interpolation = gsl_interp_alloc (gsl_interp_linear,Nzdc);
        gsl_interp_init(interpolation, interp_redshift, interp_comoving_dist, Nzdc);
        for(int64_t i=0;i<ND1;i++) {
            D1[i] = gsl_interp_eval(interpolation, interp_redshift, interp_comoving_dist, czD1[i]*inv_speed_of_light, accelerator);
        }

        if(autocorr==0) {
            for(int64_t i=0;i<ND2;i++) {
                D2[i] = gsl_interp_eval(interpolation, interp_redshift, interp_comoving_dist, czD2[i]*inv_speed_of_light, accelerator);
            }
        }
        free(interp_redshift);free(interp_comoving_dist);
        gsl_interp_free(interpolation);
        gsl_interp_accel_free(accelerator);
    }

    double *X1 = my_malloc(sizeof(*X1), ND1);
    double *Y1 = my_malloc(sizeof(*Y1), ND1);
    double *Z1 = my_malloc(sizeof(*Z1), ND1);
    if(X1 == NULL || Y1 == NULL || Z1 == NULL) {
        free(X1);free(Y1);free(Z1);
        return EXIT_FAILURE;
    }
    for(int64_t i=0;i<ND1;i++) {
        X1[i] = D1[i]*COSD(dec1[i])*COSD(ra1[i]);
        Y1[i] = D1[i]*COSD(dec1[i])*SIND(ra1[i]);
        Z1[i] = D1[i]*SIND(dec1[i]);
    }

    double *X2,*Y2,*Z2;
    if(autocorr==0) {
        X2 = my_malloc(sizeof(*X2), ND2);
        Y2 = my_malloc(sizeof(*Y2), ND2);
        Z2 = my_malloc(sizeof(*Z2), ND2);
        for(int64_t i=0;i<ND2;i++) {
            X2[i] = D2[i]*COSD(dec2[i])*COSD(ra2[i]);
            Y2[i] = D2[i]*COSD(dec2[i])*SIND(ra2[i]);
            Z2[i] = D2[i]*SIND(dec2[i]);
        }
    } else {
        X2 = X1;
        Y2 = Y1;
        Z2 = Z1;
    }

    double supp_sqr[nsbin];
    for(int i=0; i < nsbin;i++) {
        supp_sqr[i] = supp[i]*supp[i];
    }
    const double mu_max = (double) max_mu;

    double xmin=1e10,ymin=1e10,zmin=1e10;
    double xmax=-1e10,ymax=-1e10,zmax=-1e10;
    get_max_min_data_double(ND1, X1, Y1, Z1, &xmin, &ymin, &zmin, &xmax, &ymax, &zmax);

    if(autocorr==0) {
        get_max_min_data_double(ND2, X2, Y2, Z2, &xmin, &ymin, &zmin, &xmax, &ymax, &zmax);
    }

    const double xdiff = xmax-xmin;
    const double ydiff = ymax-ymin;
    const double zdiff = zmax-zmin;
    if(get_bin_refine_scheme(options) == BINNING_DFL) {
        if(smax < 0.05*xdiff) {
            options->bin_refine_factors[0] = 1;
      }
        if(smax < 0.05*ydiff) {
            options->bin_refine_factors[1] = 1;
        }
        if(smax < 0.05*zdiff) {
            options->bin_refine_factors[2] = 1;
        }
    }

    /*---Create 3-D lattice--------------------------------------*/
    int nmesh_x=0,nmesh_y=0,nmesh_z=0;
    cellarray_mocks_index_particles_double *lattice1 = gridlink_mocks_index_particles_double(ND1, X1, Y1, Z1, D1, &(extra->weights0),
                                                                                xmin, xmax, ymin, ymax, zmin, zmax,
                                                                                smax, smax, smax,
                                                                                options->bin_refine_factors[0],
                                                                                options->bin_refine_factors[1],
                                                                                options->bin_refine_factors[2],
                                                                                &nmesh_x, &nmesh_y, &nmesh_z,
                                                                                options);
    if(lattice1 == NULL) {
        return EXIT_FAILURE;
    }

    /* If there too few cells (BOOST_CELL_THRESH is ~10), and the number of cells can be increased, then boost bin refine factor by ~1*/
    const double avg_np = ((double)ND1)/(nmesh_x*nmesh_y*nmesh_z);
    const int8_t max_nmesh = fmax(nmesh_x, fmax(nmesh_y, nmesh_z));
    if((max_nmesh <= BOOST_CELL_THRESH || avg_np >= BOOST_NUMPART_THRESH)
        && max_nmesh < options->max_cells_per_dim) {
      fprintf(stderr,"%s> gridlink seems inefficient. nmesh = (%d, %d, %d); avg_np = %.3g. ", __FUNCTION__, nmesh_x, nmesh_y, nmesh_z, avg_np);
      if(get_bin_refine_scheme(options) == BINNING_DFL) {
            fprintf(stderr,"Boosting bin refine factor - should lead to better performance\n");
            // Only boost the first two dimensions.  Prevents excessive refinement.
            for(int i=0;i<2;i++) {
              options->bin_refine_factors[i] += BOOST_BIN_REF;
            }

            free_cellarray_mocks_index_particles_double(lattice1, nmesh_x * (int64_t) nmesh_y * nmesh_z);
            lattice1 = gridlink_mocks_index_particles_double(ND1, X1, Y1, Z1, D1, &(extra->weights0),
                                                             xmin, xmax, ymin, ymax, zmin, zmax,
                                                             smax, smax, smax,
                                                             options->bin_refine_factors[0],
                                                             options->bin_refine_factors[1],
                                                             options->bin_refine_factors[2],
                                                             &nmesh_x, &nmesh_y, &nmesh_z,
                                                             options);
            if(lattice1 == NULL) {
                return EXIT_FAILURE;
            }
        } else {
            fprintf(stderr,"Boosting bin refine factor could have helped. However, since custom bin refine factors "
                  "= (%d, %d, %d) are being used - continuing with inefficient mesh\n", options->bin_refine_factors[0],
                  options->bin_refine_factors[1], options->bin_refine_factors[2]);

        }
    }

    cellarray_mocks_index_particles_double *lattice2 = NULL;
    if(autocorr==0) {
        int ngrid2_x=0,ngrid2_y=0,ngrid2_z=0;
        lattice2 = gridlink_mocks_index_particles_double(ND2, X2, Y2, Z2, D2, &(extra->weights1),
                                                         xmin, xmax,
                                                         ymin, ymax,
                                                         zmin, zmax,
                                                         smax, smax, smax,
                                                         options->bin_refine_factors[0],
                                                         options->bin_refine_factors[1],
                                                         options->bin_refine_factors[2],
                                                         &ngrid2_x, &ngrid2_y, &ngrid2_z, options);
        if(lattice2 == NULL) {
            return EXIT_FAILURE;
        }
        if( ! (nmesh_x == ngrid2_x && nmesh_y == ngrid2_y && nmesh_z == ngrid2_z) ) {
            fprintf(stderr,"Error: The two sets of 3-D lattices do not have identical bins. First has dims (%d, %d, %d) while second has (%d, %d, %d)\n",
                    nmesh_x, nmesh_y, nmesh_z, ngrid2_x, ngrid2_y, ngrid2_z);
            return EXIT_FAILURE;
        }
    } else {
        lattice2 = lattice1;
    }
    free(X1);free(Y1);free(Z1);
    if(autocorr == 0) {
        free(X2);free(Y2);free(Z2);
    }

    if(options->is_comoving_dist == 0) {
        free(D1);
        if(autocorr == 0) {
            free(D2);
        }
    }

    const int64_t totncells = (int64_t) nmesh_x * (int64_t) nmesh_y * (int64_t) nmesh_z;
    {
        int status = assign_ngb_cells_mocks_index_particles_double(lattice1, lattice2, totncells,
                                                                   options->bin_refine_factors[0], options->bin_refine_factors[1], options->bin_refine_factors[2],
                                                                   nmesh_x, nmesh_y, nmesh_z,
                                                                   autocorr);
        if(status != EXIT_SUCCESS) {
            free_cellarray_mocks_index_particles_double(lattice1, totncells);
            if(autocorr == 0) {
                free_cellarray_mocks_index_particles_double(lattice2, totncells);
            }
            free(supp);
            return EXIT_FAILURE;
        }
    }
    /*---Gridlink-variables----------------*/
    const int totnbins = (nmu_bins+1)*(nsbin+1);
    const int nprojbins = nsbin-1;
#if defined(_OPENMP)
    uint64_t **all_npairs = (uint64_t **) matrix_calloc(sizeof(uint64_t), numthreads, totnbins);
    double **all_savg = NULL;
    if(options->need_avg_sep){
        all_savg = (double **) matrix_calloc(sizeof(double),numthreads,totnbins);
    }
    double **all_weightavg = NULL;
    if(need_weightavg) {
      all_weightavg = (double **) matrix_calloc(sizeof(double),numthreads,totnbins);
    }
    double **all_projpairs = (double **) matrix_calloc(sizeof(double),numthreads,nprojbins);
    double **all_projpairs_tensor = (double **) matrix_calloc(sizeof(double),numthreads,nprojbins*nprojbins);

#else //USE_OMP
    uint64_t npairs[totnbins];
    double savg[totnbins], weightavg[totnbins], projpairs[nprojbins];
    double projpairs_tensor[nprojbins*nprojbins];

    for(int i=0; i <totnbins;i++) {
        npairs[i] = 0;
        if(options->need_avg_sep) {
            savg[i] = ZERO;
        }
        if(need_weightavg) {
            weightavg[i] = ZERO;
        }
    }
    for(int i=0;i<nprojbins;i++) {
        projpairs[i] = ZERO;
        for(int j=0;j<nprojbins;j++) {
            projpairs_tensor[i*nprojbins+j] = ZERO;
        }
    }
#endif //USE_OMP

    /* runtime dispatch - get the function pointer */
    countpairs_mocks_func_ptr_double countpairs_s_mu_mocks_function_double = countpairs_s_mu_mocks_driver_double(options);
    if(countpairs_s_mu_mocks_function_double == NULL) {
        return EXIT_FAILURE;
    }

    int interrupted=0,numdone=0, abort_status=EXIT_SUCCESS;
    if(options->verbose) {
        init_my_progressbar(totncells,&interrupted);
    }

#if defined(_OPENMP)
#pragma omp parallel shared(numdone, abort_status, interrupt_status_DDsmu_mocks_double)
    {
        const int tid = omp_get_thread_num();
        uint64_t npairs[totnbins];
        double savg[totnbins], weightavg[totnbins], projpairs[nprojbins];
        double projpairs_tensor[nprojbins*nprojbins];

        for(int i=0;i<totnbins;i++) {
            npairs[i] = 0;
            if(options->need_avg_sep) {
                savg[i] = ZERO;
            }
            if(need_weightavg) {
                weightavg[i] = ZERO;
            }
        }
        for(int i=0;i<nprojbins;i++) {
            projpairs[i] = ZERO;
            for(int j=0;j<nprojbins;j++) {
                projpairs_tensor[i*nprojbins+j] = ZERO;
            }
        }

#pragma omp for  schedule(dynamic)
#endif//USE_OMP

        /*---Loop-over-Data1-particles--------------------*/
        for(int64_t index1=0;index1<totncells;index1++) {

#if defined(_OPENMP)
#pragma omp flush (abort_status, interrupt_status_DDsmu_mocks_double)
#endif
            if(abort_status == EXIT_SUCCESS && interrupt_status_DDsmu_mocks_double == EXIT_SUCCESS) {
                //omp cancel was introduced in omp 4.0 - so this is my way of checking if loop needs to be cancelled
                /* If the verbose option is not enabled, avoid outputting anything unnecessary*/
                if(options->verbose) {
#if defined(_OPENMP)
                    if (omp_get_thread_num() == 0)
#endif
                        my_progressbar(numdone,&interrupted);

#if defined(_OPENMP)
#pragma omp atomic
#endif
                    numdone++;
                }

                const cellarray_mocks_index_particles_double *first  = &(lattice1[index1]);
                if(first->nelements == 0) {
                    continue;
                }
                double *x1 = first->x;
                double *y1 = first->y;
                double *z1 = first->z;
                double *d1 = first->cz;
                const weight_struct_double *weights1 = &(first->weights);
                const int64_t N1 = first->nelements;

                if(autocorr == 1) {
                    int same_cell = 1;
                    double *this_savg = options->need_avg_sep ? &(savg[0]):NULL;
                    double *this_weightavg = need_weightavg ? weightavg:NULL;
                    const int status = countpairs_s_mu_mocks_function_double(N1, x1, y1, z1, d1, weights1,
                                                                             N1, x1, y1, z1, d1, weights1,
                                                                             same_cell,
                                                                             options->fast_divide_and_NR_steps,
                                                                             smax, smin, nsbin,
                                                                             nmu_bins, supp_sqr, mu_max,
                                                                             this_savg, npairs, projpairs,
                                                                             projpairs_tensor,
                                                                             this_weightavg, extra->weight_method);
                    /* This actually causes a race condition under OpenMP - but mostly
                       I care that an error occurred - rather than the exact value of
                       the error status */
                    abort_status |= status;
                }

                for(int64_t ngb=0;ngb<first->num_ngb;ngb++){
                    const cellarray_mocks_index_particles_double *second = first->ngb_cells[ngb];
                    if(second->nelements == 0) {
                        continue;
                    }
                    const int same_cell = 0;
                    double *x2 = second->x;
                    double *y2 = second->y;
                    double *z2 = second->z;
                    double *d2 = second->cz;
                    const weight_struct_double *weights2 = &(second->weights);
                    const int64_t N2 = second->nelements;
                    double *this_savg = options->need_avg_sep ? &(savg[0]):NULL;
                    double *this_weightavg = need_weightavg ? weightavg:NULL;
                    const int status = countpairs_s_mu_mocks_function_double(N1, x1, y1, z1, d1, weights1,
                                                                             N2, x2, y2, z2, d2, weights2,
                                                                             same_cell,
                                                                             options->fast_divide_and_NR_steps,
                                                                             smax, smin, nsbin,
                                                                             nmu_bins, supp_sqr, mu_max,
                                                                             this_savg, npairs, projpairs,
                                                                             projpairs_tensor,
                                                                             this_weightavg, extra->weight_method);
                    /* This actually causes a race condition under OpenMP - but mostly
                       I care that an error occurred - rather than the exact value of
                       the error status */
                    abort_status |= status;
                }//loop over ngb cells
            }//abort_status check
        }//i loop over ND1 particles
#if defined(_OPENMP)
        for(int i=0;i<totnbins;i++) {
            all_npairs[tid][i] = npairs[i];
            if(options->need_avg_sep) {
                all_savg[tid][i] = savg[i];
            }
            if(need_weightavg) {
                all_weightavg[tid][i] = weightavg[i];
            }
        }
        for (int i=0;i<nprojbins;i++) {
            all_projpairs[tid][i] = projpairs[i];
            for(int j=0;j<nprojbins;j++) {
                all_projpairs_tensor[tid][i*nprojbins+j] = projpairs_tensor[i*nprojbins+j];
            }
        }
    }//close the omp parallel region
#endif//USE_OMP

    free_cellarray_mocks_index_particles_double(lattice1,totncells);
    if(autocorr == 0) {
        free_cellarray_mocks_index_particles_double(lattice2,totncells);
    }

    if(abort_status != EXIT_SUCCESS || interrupt_status_DDsmu_mocks_double != EXIT_SUCCESS) {
        /* Cleanup memory here if aborting */
        free(supp);
#if defined(_OPENMP)
        matrix_free((void **) all_npairs, numthreads);
        if(options->need_avg_sep) {
            matrix_free((void **) all_savg, numthreads);
        }
        if(need_weightavg) {
            matrix_free((void **) all_weightavg, numthreads);
        }
        matrix_free((void **) all_projpairs, numthreads);
        matrix_free((void **) all_projpairs_tensor, numthreads);
#endif
        return EXIT_FAILURE;
    }

    if(options->verbose) {
        finish_myprogressbar(&interrupted);
    }

#if defined(_OPENMP)
    uint64_t npairs[totnbins];
    double savg[totnbins], weightavg[totnbins], projpairs[nprojbins];
    double projpairs_tensor[nprojbins*nprojbins];
    for(int i=0;i<totnbins;i++) {
        npairs[i] = 0;
        if(options->need_avg_sep) {
            savg[i] = ZERO;
        }
        if(need_weightavg) {
            weightavg[i] = ZERO;
        }
    }
    for(int i=0;i<nprojbins;i++) {
        projpairs[i] = ZERO;
        for(int j=0;j<nprojbins;j++) {
            projpairs_tensor[i*nprojbins+j] = ZERO;
        }
    }

    for(int i=0;i<numthreads;i++) {
        for(int j=0;j<totnbins;j++) {
            npairs[j] += all_npairs[i][j];
            if(options->need_avg_sep) {
                savg[j] += all_savg[i][j];
            }
            if(need_weightavg) {
                weightavg[j] += all_weightavg[i][j];
            }
        }
        for(int j=0;j<nprojbins;j++) {
            projpairs[j] += all_projpairs[i][j];
            for(int k=0;k<nprojbins;k++) {
                projpairs_tensor[j*nprojbins+k] += all_projpairs_tensor[i][j*nprojbins+k];
            }
        }
    }
    matrix_free((void **) all_npairs, numthreads);
    if(options->need_avg_sep) {
        matrix_free((void **) all_savg, numthreads);
    }
    if(need_weightavg) {
        matrix_free((void **) all_weightavg, numthreads);
    }
    matrix_free((void **) all_projpairs, numthreads);
    matrix_free((void **) all_projpairs_tensor, numthreads);

#endif //USE_OMP

    //The code does not double count for autocorrelations
    //which means the npairs and savg values need to be doubled;
    if(autocorr == 1) {
        const uint64_t int_fac = 2;
        const double dbl_fac = (double) 2.0;
        for(int i=0;i<totnbins;i++) {
            npairs[i] *= int_fac;
            if(options->need_avg_sep) {
                savg[i] *= dbl_fac;
            }
            if(need_weightavg) {
                weightavg[i] *= dbl_fac;
            }
        }
        //TODO: do i also want to double this? think so
        for(int i=0;i<nprojbins;i++) {
            projpairs[i] *= dbl_fac;
            for(int j=0;j<nprojbins;j++) {
                projpairs_tensor[i*nprojbins+j] *= dbl_fac;
            }
        }

    }

    for(int i=0;i<totnbins;i++) {
        if(npairs[i] > 0) {
            if(options->need_avg_sep) {
                savg[i] /= (double) npairs[i] ;
            }
            if(need_weightavg) {
                weightavg[i] /= (double) npairs[i];
            }
        }
    }
    // don't need proj_pairs here, not averaging

    results->nsbin   = nsbin;
    results->nmu_bins = nmu_bins;
    results->mu_max = max_mu;//NOTE max_mu which is double and not mu_max (which might be float)
    results->mu_min = ZERO;
    results->npairs = my_malloc(sizeof(*(results->npairs)), totnbins);
    results->projpairs = my_malloc(sizeof(*(results->npairs)), nprojbins);
    results->projpairs_tensor = my_malloc(sizeof(*(results->npairs)), nprojbins*nprojbins);
    results->supp   = my_malloc(sizeof(*(results->supp))  , nsbin);
    results->savg  = my_malloc(sizeof(*(results->savg)) , totnbins);
    results->weightavg  = my_calloc(sizeof(double)  , totnbins);
    if(results->npairs == NULL || results->supp == NULL || results->savg == NULL || results->weightavg == NULL || results->projpairs == NULL) {
        free_results_mocks_s_mu(results);
        free(supp);
        return EXIT_FAILURE;
    }

    for(int i=0;i<nsbin;i++) {
        results->supp[i] = supp[i];
        for(int j=0;j<nmu_bins;j++) {
            const int index = i*(nmu_bins+1) + j;
            if( index >= totnbins ) {
                fprintf(stderr, "ERROR: In %s> index = %d must be in range [0, %d)\n", __FUNCTION__, index, totnbins);
                free_results_mocks_s_mu(results);
                free(supp);
                return EXIT_FAILURE;
            }
            results->npairs[index] = npairs[index];
            results->savg[index] = ZERO;
            results->weightavg[index] = ZERO;
            if(options->need_avg_sep) {
                results->savg[index] = savg[index];
            }
            if(need_weightavg) {
                results->weightavg[index] = weightavg[index];
            }
        }
    }
    for(int i=0;i<nprojbins;i++) {
        results->projpairs[i] = projpairs[i];
        for(int j=0;j<nprojbins;j++) {
                results->projpairs_tensor[i*nprojbins+j] = projpairs_tensor[i*nprojbins+j];
            }
    }
    free(supp);

    /* reset interrupt handlers to default */
    RESET_INTERRUPT_HANDLERS();
    reset_bin_refine_factors(options);

    if(options->c_api_timer) {
        struct timeval t1;
        gettimeofday(&t1, NULL);
        options->c_api_time = ADD_DIFF_TIME(t0, t1);
    }

    return EXIT_SUCCESS;
}


/*
 * Copyright (c) 1997-1999 Massachusetts Institute of Technology
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 *
 */

/* This file was automatically generated --- DO NOT EDIT */
/* Generated on Mon Mar  8 17:46:56 EST 1999 */

#include <fftw-int.h>
#include <fftw.h>

/* Generated by: ./genfft -magic-alignment-check -magic-twiddle-load-all -magic-variables 4 -magic-loopi -twiddleinv 32 */

/*
 * This function contains 434 FP additions, 208 FP multiplications,
 * (or, 340 additions, 114 multiplications, 94 fused multiply/add),
 * 90 stack variables, and 128 memory accesses
 */
static const fftw_real K555570233 = FFTW_KONST(+0.555570233019602224742830813948532874374937191);
static const fftw_real K831469612 = FFTW_KONST(+0.831469612302545237078788377617905756738560812);
static const fftw_real K980785280 = FFTW_KONST(+0.980785280403230449126182236134239036973933731);
static const fftw_real K195090322 = FFTW_KONST(+0.195090322016128267848284868477022240927691618);
static const fftw_real K923879532 = FFTW_KONST(+0.923879532511286756128183189396788286822416626);
static const fftw_real K382683432 = FFTW_KONST(+0.382683432365089771728459984030398866761344562);
static const fftw_real K707106781 = FFTW_KONST(+0.707106781186547524400844362104849039284835938);

/*
 * Generator Id's :
 * $Id: exprdag.ml,v 1.36 1999/02/19 17:22:11 athena Exp $
 * $Id: fft.ml,v 1.41 1999/02/19 17:22:13 athena Exp $
 * $Id: to_c.ml,v 1.24 1999/02/19 17:22:17 athena Exp $
 */

void fftwi_twiddle_32(fftw_complex *A, const fftw_complex *W, int iostride, int m, int dist)
{
     int i;
     fftw_complex *inout;
     inout = A;
     for (i = m; i > 0; i = i - 1, inout = inout + dist, W = W + 31) {
	  fftw_real tmp19;
	  fftw_real tmp387;
	  fftw_real tmp472;
	  fftw_real tmp486;
	  fftw_real tmp442;
	  fftw_real tmp456;
	  fftw_real tmp191;
	  fftw_real tmp303;
	  fftw_real tmp161;
	  fftw_real tmp403;
	  fftw_real tmp276;
	  fftw_real tmp316;
	  fftw_real tmp372;
	  fftw_real tmp400;
	  fftw_real tmp259;
	  fftw_real tmp319;
	  fftw_real tmp42;
	  fftw_real tmp455;
	  fftw_real tmp201;
	  fftw_real tmp304;
	  fftw_real tmp390;
	  fftw_real tmp437;
	  fftw_real tmp196;
	  fftw_real tmp305;
	  fftw_real tmp184;
	  fftw_real tmp401;
	  fftw_real tmp375;
	  fftw_real tmp404;
	  fftw_real tmp270;
	  fftw_real tmp317;
	  fftw_real tmp279;
	  fftw_real tmp320;
	  fftw_real tmp66;
	  fftw_real tmp395;
	  fftw_real tmp224;
	  fftw_real tmp312;
	  fftw_real tmp357;
	  fftw_real tmp396;
	  fftw_real tmp219;
	  fftw_real tmp311;
	  fftw_real tmp114;
	  fftw_real tmp410;
	  fftw_real tmp249;
	  fftw_real tmp323;
	  fftw_real tmp363;
	  fftw_real tmp407;
	  fftw_real tmp232;
	  fftw_real tmp326;
	  fftw_real tmp89;
	  fftw_real tmp393;
	  fftw_real tmp213;
	  fftw_real tmp309;
	  fftw_real tmp354;
	  fftw_real tmp392;
	  fftw_real tmp208;
	  fftw_real tmp308;
	  fftw_real tmp137;
	  fftw_real tmp408;
	  fftw_real tmp366;
	  fftw_real tmp411;
	  fftw_real tmp243;
	  fftw_real tmp324;
	  fftw_real tmp252;
	  fftw_real tmp327;
	  ASSERT_ALIGNED_DOUBLE();
	  {
	       fftw_real tmp1;
	       fftw_real tmp440;
	       fftw_real tmp6;
	       fftw_real tmp439;
	       fftw_real tmp12;
	       fftw_real tmp188;
	       fftw_real tmp17;
	       fftw_real tmp189;
	       ASSERT_ALIGNED_DOUBLE();
	       tmp1 = c_re(inout[0]);
	       tmp440 = c_im(inout[0]);
	       {
		    fftw_real tmp3;
		    fftw_real tmp5;
		    fftw_real tmp2;
		    fftw_real tmp4;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp3 = c_re(inout[16 * iostride]);
		    tmp5 = c_im(inout[16 * iostride]);
		    tmp2 = c_re(W[15]);
		    tmp4 = c_im(W[15]);
		    tmp6 = (tmp2 * tmp3) + (tmp4 * tmp5);
		    tmp439 = (tmp2 * tmp5) - (tmp4 * tmp3);
	       }
	       {
		    fftw_real tmp9;
		    fftw_real tmp11;
		    fftw_real tmp8;
		    fftw_real tmp10;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp9 = c_re(inout[8 * iostride]);
		    tmp11 = c_im(inout[8 * iostride]);
		    tmp8 = c_re(W[7]);
		    tmp10 = c_im(W[7]);
		    tmp12 = (tmp8 * tmp9) + (tmp10 * tmp11);
		    tmp188 = (tmp8 * tmp11) - (tmp10 * tmp9);
	       }
	       {
		    fftw_real tmp14;
		    fftw_real tmp16;
		    fftw_real tmp13;
		    fftw_real tmp15;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp14 = c_re(inout[24 * iostride]);
		    tmp16 = c_im(inout[24 * iostride]);
		    tmp13 = c_re(W[23]);
		    tmp15 = c_im(W[23]);
		    tmp17 = (tmp13 * tmp14) + (tmp15 * tmp16);
		    tmp189 = (tmp13 * tmp16) - (tmp15 * tmp14);
	       }
	       {
		    fftw_real tmp7;
		    fftw_real tmp18;
		    fftw_real tmp470;
		    fftw_real tmp471;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp7 = tmp1 + tmp6;
		    tmp18 = tmp12 + tmp17;
		    tmp19 = tmp7 + tmp18;
		    tmp387 = tmp7 - tmp18;
		    tmp470 = tmp12 - tmp17;
		    tmp471 = tmp440 - tmp439;
		    tmp472 = tmp470 + tmp471;
		    tmp486 = tmp471 - tmp470;
	       }
	       {
		    fftw_real tmp438;
		    fftw_real tmp441;
		    fftw_real tmp187;
		    fftw_real tmp190;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp438 = tmp188 + tmp189;
		    tmp441 = tmp439 + tmp440;
		    tmp442 = tmp438 + tmp441;
		    tmp456 = tmp441 - tmp438;
		    tmp187 = tmp1 - tmp6;
		    tmp190 = tmp188 - tmp189;
		    tmp191 = tmp187 - tmp190;
		    tmp303 = tmp187 + tmp190;
	       }
	  }
	  {
	       fftw_real tmp143;
	       fftw_real tmp272;
	       fftw_real tmp159;
	       fftw_real tmp257;
	       fftw_real tmp148;
	       fftw_real tmp273;
	       fftw_real tmp154;
	       fftw_real tmp256;
	       ASSERT_ALIGNED_DOUBLE();
	       {
		    fftw_real tmp140;
		    fftw_real tmp142;
		    fftw_real tmp139;
		    fftw_real tmp141;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp140 = c_re(inout[31 * iostride]);
		    tmp142 = c_im(inout[31 * iostride]);
		    tmp139 = c_re(W[30]);
		    tmp141 = c_im(W[30]);
		    tmp143 = (tmp139 * tmp140) + (tmp141 * tmp142);
		    tmp272 = (tmp139 * tmp142) - (tmp141 * tmp140);
	       }
	       {
		    fftw_real tmp156;
		    fftw_real tmp158;
		    fftw_real tmp155;
		    fftw_real tmp157;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp156 = c_re(inout[23 * iostride]);
		    tmp158 = c_im(inout[23 * iostride]);
		    tmp155 = c_re(W[22]);
		    tmp157 = c_im(W[22]);
		    tmp159 = (tmp155 * tmp156) + (tmp157 * tmp158);
		    tmp257 = (tmp155 * tmp158) - (tmp157 * tmp156);
	       }
	       {
		    fftw_real tmp145;
		    fftw_real tmp147;
		    fftw_real tmp144;
		    fftw_real tmp146;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp145 = c_re(inout[15 * iostride]);
		    tmp147 = c_im(inout[15 * iostride]);
		    tmp144 = c_re(W[14]);
		    tmp146 = c_im(W[14]);
		    tmp148 = (tmp144 * tmp145) + (tmp146 * tmp147);
		    tmp273 = (tmp144 * tmp147) - (tmp146 * tmp145);
	       }
	       {
		    fftw_real tmp151;
		    fftw_real tmp153;
		    fftw_real tmp150;
		    fftw_real tmp152;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp151 = c_re(inout[7 * iostride]);
		    tmp153 = c_im(inout[7 * iostride]);
		    tmp150 = c_re(W[6]);
		    tmp152 = c_im(W[6]);
		    tmp154 = (tmp150 * tmp151) + (tmp152 * tmp153);
		    tmp256 = (tmp150 * tmp153) - (tmp152 * tmp151);
	       }
	       {
		    fftw_real tmp149;
		    fftw_real tmp160;
		    fftw_real tmp274;
		    fftw_real tmp275;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp149 = tmp143 + tmp148;
		    tmp160 = tmp154 + tmp159;
		    tmp161 = tmp149 + tmp160;
		    tmp403 = tmp149 - tmp160;
		    tmp274 = tmp272 - tmp273;
		    tmp275 = tmp154 - tmp159;
		    tmp276 = tmp274 + tmp275;
		    tmp316 = tmp274 - tmp275;
	       }
	       {
		    fftw_real tmp370;
		    fftw_real tmp371;
		    fftw_real tmp255;
		    fftw_real tmp258;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp370 = tmp272 + tmp273;
		    tmp371 = tmp256 + tmp257;
		    tmp372 = tmp370 + tmp371;
		    tmp400 = tmp370 - tmp371;
		    tmp255 = tmp143 - tmp148;
		    tmp258 = tmp256 - tmp257;
		    tmp259 = tmp255 - tmp258;
		    tmp319 = tmp255 + tmp258;
	       }
	  }
	  {
	       fftw_real tmp24;
	       fftw_real tmp193;
	       fftw_real tmp40;
	       fftw_real tmp199;
	       fftw_real tmp29;
	       fftw_real tmp194;
	       fftw_real tmp35;
	       fftw_real tmp198;
	       ASSERT_ALIGNED_DOUBLE();
	       {
		    fftw_real tmp21;
		    fftw_real tmp23;
		    fftw_real tmp20;
		    fftw_real tmp22;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp21 = c_re(inout[4 * iostride]);
		    tmp23 = c_im(inout[4 * iostride]);
		    tmp20 = c_re(W[3]);
		    tmp22 = c_im(W[3]);
		    tmp24 = (tmp20 * tmp21) + (tmp22 * tmp23);
		    tmp193 = (tmp20 * tmp23) - (tmp22 * tmp21);
	       }
	       {
		    fftw_real tmp37;
		    fftw_real tmp39;
		    fftw_real tmp36;
		    fftw_real tmp38;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp37 = c_re(inout[12 * iostride]);
		    tmp39 = c_im(inout[12 * iostride]);
		    tmp36 = c_re(W[11]);
		    tmp38 = c_im(W[11]);
		    tmp40 = (tmp36 * tmp37) + (tmp38 * tmp39);
		    tmp199 = (tmp36 * tmp39) - (tmp38 * tmp37);
	       }
	       {
		    fftw_real tmp26;
		    fftw_real tmp28;
		    fftw_real tmp25;
		    fftw_real tmp27;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp26 = c_re(inout[20 * iostride]);
		    tmp28 = c_im(inout[20 * iostride]);
		    tmp25 = c_re(W[19]);
		    tmp27 = c_im(W[19]);
		    tmp29 = (tmp25 * tmp26) + (tmp27 * tmp28);
		    tmp194 = (tmp25 * tmp28) - (tmp27 * tmp26);
	       }
	       {
		    fftw_real tmp32;
		    fftw_real tmp34;
		    fftw_real tmp31;
		    fftw_real tmp33;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp32 = c_re(inout[28 * iostride]);
		    tmp34 = c_im(inout[28 * iostride]);
		    tmp31 = c_re(W[27]);
		    tmp33 = c_im(W[27]);
		    tmp35 = (tmp31 * tmp32) + (tmp33 * tmp34);
		    tmp198 = (tmp31 * tmp34) - (tmp33 * tmp32);
	       }
	       {
		    fftw_real tmp30;
		    fftw_real tmp41;
		    fftw_real tmp197;
		    fftw_real tmp200;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp30 = tmp24 + tmp29;
		    tmp41 = tmp35 + tmp40;
		    tmp42 = tmp30 + tmp41;
		    tmp455 = tmp30 - tmp41;
		    tmp197 = tmp35 - tmp40;
		    tmp200 = tmp198 - tmp199;
		    tmp201 = tmp197 + tmp200;
		    tmp304 = tmp200 - tmp197;
	       }
	       {
		    fftw_real tmp388;
		    fftw_real tmp389;
		    fftw_real tmp192;
		    fftw_real tmp195;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp388 = tmp198 + tmp199;
		    tmp389 = tmp193 + tmp194;
		    tmp390 = tmp388 - tmp389;
		    tmp437 = tmp389 + tmp388;
		    tmp192 = tmp24 - tmp29;
		    tmp195 = tmp193 - tmp194;
		    tmp196 = tmp192 - tmp195;
		    tmp305 = tmp192 + tmp195;
	       }
	  }
	  {
	       fftw_real tmp166;
	       fftw_real tmp261;
	       fftw_real tmp171;
	       fftw_real tmp262;
	       fftw_real tmp260;
	       fftw_real tmp263;
	       fftw_real tmp177;
	       fftw_real tmp266;
	       fftw_real tmp182;
	       fftw_real tmp267;
	       fftw_real tmp265;
	       fftw_real tmp268;
	       ASSERT_ALIGNED_DOUBLE();
	       {
		    fftw_real tmp163;
		    fftw_real tmp165;
		    fftw_real tmp162;
		    fftw_real tmp164;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp163 = c_re(inout[3 * iostride]);
		    tmp165 = c_im(inout[3 * iostride]);
		    tmp162 = c_re(W[2]);
		    tmp164 = c_im(W[2]);
		    tmp166 = (tmp162 * tmp163) + (tmp164 * tmp165);
		    tmp261 = (tmp162 * tmp165) - (tmp164 * tmp163);
	       }
	       {
		    fftw_real tmp168;
		    fftw_real tmp170;
		    fftw_real tmp167;
		    fftw_real tmp169;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp168 = c_re(inout[19 * iostride]);
		    tmp170 = c_im(inout[19 * iostride]);
		    tmp167 = c_re(W[18]);
		    tmp169 = c_im(W[18]);
		    tmp171 = (tmp167 * tmp168) + (tmp169 * tmp170);
		    tmp262 = (tmp167 * tmp170) - (tmp169 * tmp168);
	       }
	       tmp260 = tmp166 - tmp171;
	       tmp263 = tmp261 - tmp262;
	       {
		    fftw_real tmp174;
		    fftw_real tmp176;
		    fftw_real tmp173;
		    fftw_real tmp175;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp174 = c_re(inout[27 * iostride]);
		    tmp176 = c_im(inout[27 * iostride]);
		    tmp173 = c_re(W[26]);
		    tmp175 = c_im(W[26]);
		    tmp177 = (tmp173 * tmp174) + (tmp175 * tmp176);
		    tmp266 = (tmp173 * tmp176) - (tmp175 * tmp174);
	       }
	       {
		    fftw_real tmp179;
		    fftw_real tmp181;
		    fftw_real tmp178;
		    fftw_real tmp180;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp179 = c_re(inout[11 * iostride]);
		    tmp181 = c_im(inout[11 * iostride]);
		    tmp178 = c_re(W[10]);
		    tmp180 = c_im(W[10]);
		    tmp182 = (tmp178 * tmp179) + (tmp180 * tmp181);
		    tmp267 = (tmp178 * tmp181) - (tmp180 * tmp179);
	       }
	       tmp265 = tmp177 - tmp182;
	       tmp268 = tmp266 - tmp267;
	       {
		    fftw_real tmp172;
		    fftw_real tmp183;
		    fftw_real tmp373;
		    fftw_real tmp374;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp172 = tmp166 + tmp171;
		    tmp183 = tmp177 + tmp182;
		    tmp184 = tmp172 + tmp183;
		    tmp401 = tmp172 - tmp183;
		    tmp373 = tmp261 + tmp262;
		    tmp374 = tmp266 + tmp267;
		    tmp375 = tmp373 + tmp374;
		    tmp404 = tmp374 - tmp373;
	       }
	       {
		    fftw_real tmp264;
		    fftw_real tmp269;
		    fftw_real tmp277;
		    fftw_real tmp278;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp264 = tmp260 - tmp263;
		    tmp269 = tmp265 + tmp268;
		    tmp270 = K707106781 * (tmp264 + tmp269);
		    tmp317 = K707106781 * (tmp264 - tmp269);
		    tmp277 = tmp260 + tmp263;
		    tmp278 = tmp268 - tmp265;
		    tmp279 = K707106781 * (tmp277 + tmp278);
		    tmp320 = K707106781 * (tmp278 - tmp277);
	       }
	  }
	  {
	       fftw_real tmp48;
	       fftw_real tmp215;
	       fftw_real tmp64;
	       fftw_real tmp222;
	       fftw_real tmp53;
	       fftw_real tmp216;
	       fftw_real tmp59;
	       fftw_real tmp221;
	       ASSERT_ALIGNED_DOUBLE();
	       {
		    fftw_real tmp45;
		    fftw_real tmp47;
		    fftw_real tmp44;
		    fftw_real tmp46;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp45 = c_re(inout[2 * iostride]);
		    tmp47 = c_im(inout[2 * iostride]);
		    tmp44 = c_re(W[1]);
		    tmp46 = c_im(W[1]);
		    tmp48 = (tmp44 * tmp45) + (tmp46 * tmp47);
		    tmp215 = (tmp44 * tmp47) - (tmp46 * tmp45);
	       }
	       {
		    fftw_real tmp61;
		    fftw_real tmp63;
		    fftw_real tmp60;
		    fftw_real tmp62;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp61 = c_re(inout[26 * iostride]);
		    tmp63 = c_im(inout[26 * iostride]);
		    tmp60 = c_re(W[25]);
		    tmp62 = c_im(W[25]);
		    tmp64 = (tmp60 * tmp61) + (tmp62 * tmp63);
		    tmp222 = (tmp60 * tmp63) - (tmp62 * tmp61);
	       }
	       {
		    fftw_real tmp50;
		    fftw_real tmp52;
		    fftw_real tmp49;
		    fftw_real tmp51;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp50 = c_re(inout[18 * iostride]);
		    tmp52 = c_im(inout[18 * iostride]);
		    tmp49 = c_re(W[17]);
		    tmp51 = c_im(W[17]);
		    tmp53 = (tmp49 * tmp50) + (tmp51 * tmp52);
		    tmp216 = (tmp49 * tmp52) - (tmp51 * tmp50);
	       }
	       {
		    fftw_real tmp56;
		    fftw_real tmp58;
		    fftw_real tmp55;
		    fftw_real tmp57;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp56 = c_re(inout[10 * iostride]);
		    tmp58 = c_im(inout[10 * iostride]);
		    tmp55 = c_re(W[9]);
		    tmp57 = c_im(W[9]);
		    tmp59 = (tmp55 * tmp56) + (tmp57 * tmp58);
		    tmp221 = (tmp55 * tmp58) - (tmp57 * tmp56);
	       }
	       {
		    fftw_real tmp54;
		    fftw_real tmp65;
		    fftw_real tmp220;
		    fftw_real tmp223;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp54 = tmp48 + tmp53;
		    tmp65 = tmp59 + tmp64;
		    tmp66 = tmp54 + tmp65;
		    tmp395 = tmp54 - tmp65;
		    tmp220 = tmp48 - tmp53;
		    tmp223 = tmp221 - tmp222;
		    tmp224 = tmp220 - tmp223;
		    tmp312 = tmp220 + tmp223;
	       }
	       {
		    fftw_real tmp355;
		    fftw_real tmp356;
		    fftw_real tmp217;
		    fftw_real tmp218;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp355 = tmp215 + tmp216;
		    tmp356 = tmp221 + tmp222;
		    tmp357 = tmp355 + tmp356;
		    tmp396 = tmp355 - tmp356;
		    tmp217 = tmp215 - tmp216;
		    tmp218 = tmp59 - tmp64;
		    tmp219 = tmp217 + tmp218;
		    tmp311 = tmp217 - tmp218;
	       }
	  }
	  {
	       fftw_real tmp96;
	       fftw_real tmp245;
	       fftw_real tmp112;
	       fftw_real tmp230;
	       fftw_real tmp101;
	       fftw_real tmp246;
	       fftw_real tmp107;
	       fftw_real tmp229;
	       ASSERT_ALIGNED_DOUBLE();
	       {
		    fftw_real tmp93;
		    fftw_real tmp95;
		    fftw_real tmp92;
		    fftw_real tmp94;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp93 = c_re(inout[iostride]);
		    tmp95 = c_im(inout[iostride]);
		    tmp92 = c_re(W[0]);
		    tmp94 = c_im(W[0]);
		    tmp96 = (tmp92 * tmp93) + (tmp94 * tmp95);
		    tmp245 = (tmp92 * tmp95) - (tmp94 * tmp93);
	       }
	       {
		    fftw_real tmp109;
		    fftw_real tmp111;
		    fftw_real tmp108;
		    fftw_real tmp110;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp109 = c_re(inout[25 * iostride]);
		    tmp111 = c_im(inout[25 * iostride]);
		    tmp108 = c_re(W[24]);
		    tmp110 = c_im(W[24]);
		    tmp112 = (tmp108 * tmp109) + (tmp110 * tmp111);
		    tmp230 = (tmp108 * tmp111) - (tmp110 * tmp109);
	       }
	       {
		    fftw_real tmp98;
		    fftw_real tmp100;
		    fftw_real tmp97;
		    fftw_real tmp99;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp98 = c_re(inout[17 * iostride]);
		    tmp100 = c_im(inout[17 * iostride]);
		    tmp97 = c_re(W[16]);
		    tmp99 = c_im(W[16]);
		    tmp101 = (tmp97 * tmp98) + (tmp99 * tmp100);
		    tmp246 = (tmp97 * tmp100) - (tmp99 * tmp98);
	       }
	       {
		    fftw_real tmp104;
		    fftw_real tmp106;
		    fftw_real tmp103;
		    fftw_real tmp105;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp104 = c_re(inout[9 * iostride]);
		    tmp106 = c_im(inout[9 * iostride]);
		    tmp103 = c_re(W[8]);
		    tmp105 = c_im(W[8]);
		    tmp107 = (tmp103 * tmp104) + (tmp105 * tmp106);
		    tmp229 = (tmp103 * tmp106) - (tmp105 * tmp104);
	       }
	       {
		    fftw_real tmp102;
		    fftw_real tmp113;
		    fftw_real tmp247;
		    fftw_real tmp248;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp102 = tmp96 + tmp101;
		    tmp113 = tmp107 + tmp112;
		    tmp114 = tmp102 + tmp113;
		    tmp410 = tmp102 - tmp113;
		    tmp247 = tmp245 - tmp246;
		    tmp248 = tmp107 - tmp112;
		    tmp249 = tmp247 + tmp248;
		    tmp323 = tmp247 - tmp248;
	       }
	       {
		    fftw_real tmp361;
		    fftw_real tmp362;
		    fftw_real tmp228;
		    fftw_real tmp231;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp361 = tmp245 + tmp246;
		    tmp362 = tmp229 + tmp230;
		    tmp363 = tmp361 + tmp362;
		    tmp407 = tmp361 - tmp362;
		    tmp228 = tmp96 - tmp101;
		    tmp231 = tmp229 - tmp230;
		    tmp232 = tmp228 - tmp231;
		    tmp326 = tmp228 + tmp231;
	       }
	  }
	  {
	       fftw_real tmp71;
	       fftw_real tmp204;
	       fftw_real tmp87;
	       fftw_real tmp211;
	       fftw_real tmp76;
	       fftw_real tmp205;
	       fftw_real tmp82;
	       fftw_real tmp210;
	       ASSERT_ALIGNED_DOUBLE();
	       {
		    fftw_real tmp68;
		    fftw_real tmp70;
		    fftw_real tmp67;
		    fftw_real tmp69;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp68 = c_re(inout[30 * iostride]);
		    tmp70 = c_im(inout[30 * iostride]);
		    tmp67 = c_re(W[29]);
		    tmp69 = c_im(W[29]);
		    tmp71 = (tmp67 * tmp68) + (tmp69 * tmp70);
		    tmp204 = (tmp67 * tmp70) - (tmp69 * tmp68);
	       }
	       {
		    fftw_real tmp84;
		    fftw_real tmp86;
		    fftw_real tmp83;
		    fftw_real tmp85;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp84 = c_re(inout[22 * iostride]);
		    tmp86 = c_im(inout[22 * iostride]);
		    tmp83 = c_re(W[21]);
		    tmp85 = c_im(W[21]);
		    tmp87 = (tmp83 * tmp84) + (tmp85 * tmp86);
		    tmp211 = (tmp83 * tmp86) - (tmp85 * tmp84);
	       }
	       {
		    fftw_real tmp73;
		    fftw_real tmp75;
		    fftw_real tmp72;
		    fftw_real tmp74;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp73 = c_re(inout[14 * iostride]);
		    tmp75 = c_im(inout[14 * iostride]);
		    tmp72 = c_re(W[13]);
		    tmp74 = c_im(W[13]);
		    tmp76 = (tmp72 * tmp73) + (tmp74 * tmp75);
		    tmp205 = (tmp72 * tmp75) - (tmp74 * tmp73);
	       }
	       {
		    fftw_real tmp79;
		    fftw_real tmp81;
		    fftw_real tmp78;
		    fftw_real tmp80;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp79 = c_re(inout[6 * iostride]);
		    tmp81 = c_im(inout[6 * iostride]);
		    tmp78 = c_re(W[5]);
		    tmp80 = c_im(W[5]);
		    tmp82 = (tmp78 * tmp79) + (tmp80 * tmp81);
		    tmp210 = (tmp78 * tmp81) - (tmp80 * tmp79);
	       }
	       {
		    fftw_real tmp77;
		    fftw_real tmp88;
		    fftw_real tmp209;
		    fftw_real tmp212;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp77 = tmp71 + tmp76;
		    tmp88 = tmp82 + tmp87;
		    tmp89 = tmp77 + tmp88;
		    tmp393 = tmp77 - tmp88;
		    tmp209 = tmp71 - tmp76;
		    tmp212 = tmp210 - tmp211;
		    tmp213 = tmp209 - tmp212;
		    tmp309 = tmp209 + tmp212;
	       }
	       {
		    fftw_real tmp352;
		    fftw_real tmp353;
		    fftw_real tmp206;
		    fftw_real tmp207;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp352 = tmp204 + tmp205;
		    tmp353 = tmp210 + tmp211;
		    tmp354 = tmp352 + tmp353;
		    tmp392 = tmp352 - tmp353;
		    tmp206 = tmp204 - tmp205;
		    tmp207 = tmp82 - tmp87;
		    tmp208 = tmp206 + tmp207;
		    tmp308 = tmp206 - tmp207;
	       }
	  }
	  {
	       fftw_real tmp119;
	       fftw_real tmp234;
	       fftw_real tmp124;
	       fftw_real tmp235;
	       fftw_real tmp233;
	       fftw_real tmp236;
	       fftw_real tmp130;
	       fftw_real tmp239;
	       fftw_real tmp135;
	       fftw_real tmp240;
	       fftw_real tmp238;
	       fftw_real tmp241;
	       ASSERT_ALIGNED_DOUBLE();
	       {
		    fftw_real tmp116;
		    fftw_real tmp118;
		    fftw_real tmp115;
		    fftw_real tmp117;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp116 = c_re(inout[5 * iostride]);
		    tmp118 = c_im(inout[5 * iostride]);
		    tmp115 = c_re(W[4]);
		    tmp117 = c_im(W[4]);
		    tmp119 = (tmp115 * tmp116) + (tmp117 * tmp118);
		    tmp234 = (tmp115 * tmp118) - (tmp117 * tmp116);
	       }
	       {
		    fftw_real tmp121;
		    fftw_real tmp123;
		    fftw_real tmp120;
		    fftw_real tmp122;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp121 = c_re(inout[21 * iostride]);
		    tmp123 = c_im(inout[21 * iostride]);
		    tmp120 = c_re(W[20]);
		    tmp122 = c_im(W[20]);
		    tmp124 = (tmp120 * tmp121) + (tmp122 * tmp123);
		    tmp235 = (tmp120 * tmp123) - (tmp122 * tmp121);
	       }
	       tmp233 = tmp119 - tmp124;
	       tmp236 = tmp234 - tmp235;
	       {
		    fftw_real tmp127;
		    fftw_real tmp129;
		    fftw_real tmp126;
		    fftw_real tmp128;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp127 = c_re(inout[29 * iostride]);
		    tmp129 = c_im(inout[29 * iostride]);
		    tmp126 = c_re(W[28]);
		    tmp128 = c_im(W[28]);
		    tmp130 = (tmp126 * tmp127) + (tmp128 * tmp129);
		    tmp239 = (tmp126 * tmp129) - (tmp128 * tmp127);
	       }
	       {
		    fftw_real tmp132;
		    fftw_real tmp134;
		    fftw_real tmp131;
		    fftw_real tmp133;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp132 = c_re(inout[13 * iostride]);
		    tmp134 = c_im(inout[13 * iostride]);
		    tmp131 = c_re(W[12]);
		    tmp133 = c_im(W[12]);
		    tmp135 = (tmp131 * tmp132) + (tmp133 * tmp134);
		    tmp240 = (tmp131 * tmp134) - (tmp133 * tmp132);
	       }
	       tmp238 = tmp130 - tmp135;
	       tmp241 = tmp239 - tmp240;
	       {
		    fftw_real tmp125;
		    fftw_real tmp136;
		    fftw_real tmp364;
		    fftw_real tmp365;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp125 = tmp119 + tmp124;
		    tmp136 = tmp130 + tmp135;
		    tmp137 = tmp125 + tmp136;
		    tmp408 = tmp125 - tmp136;
		    tmp364 = tmp234 + tmp235;
		    tmp365 = tmp239 + tmp240;
		    tmp366 = tmp364 + tmp365;
		    tmp411 = tmp365 - tmp364;
	       }
	       {
		    fftw_real tmp237;
		    fftw_real tmp242;
		    fftw_real tmp250;
		    fftw_real tmp251;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp237 = tmp233 - tmp236;
		    tmp242 = tmp238 + tmp241;
		    tmp243 = K707106781 * (tmp237 + tmp242);
		    tmp324 = K707106781 * (tmp237 - tmp242);
		    tmp250 = tmp233 + tmp236;
		    tmp251 = tmp241 - tmp238;
		    tmp252 = K707106781 * (tmp250 + tmp251);
		    tmp327 = K707106781 * (tmp251 - tmp250);
	       }
	  }
	  {
	       fftw_real tmp91;
	       fftw_real tmp383;
	       fftw_real tmp444;
	       fftw_real tmp446;
	       fftw_real tmp186;
	       fftw_real tmp445;
	       fftw_real tmp386;
	       fftw_real tmp435;
	       ASSERT_ALIGNED_DOUBLE();
	       {
		    fftw_real tmp43;
		    fftw_real tmp90;
		    fftw_real tmp436;
		    fftw_real tmp443;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp43 = tmp19 + tmp42;
		    tmp90 = tmp66 + tmp89;
		    tmp91 = tmp43 + tmp90;
		    tmp383 = tmp43 - tmp90;
		    tmp436 = tmp357 + tmp354;
		    tmp443 = tmp437 + tmp442;
		    tmp444 = tmp436 + tmp443;
		    tmp446 = tmp443 - tmp436;
	       }
	       {
		    fftw_real tmp138;
		    fftw_real tmp185;
		    fftw_real tmp384;
		    fftw_real tmp385;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp138 = tmp114 + tmp137;
		    tmp185 = tmp161 + tmp184;
		    tmp186 = tmp138 + tmp185;
		    tmp445 = tmp138 - tmp185;
		    tmp384 = tmp372 + tmp375;
		    tmp385 = tmp363 + tmp366;
		    tmp386 = tmp384 - tmp385;
		    tmp435 = tmp385 + tmp384;
	       }
	       c_re(inout[16 * iostride]) = tmp91 - tmp186;
	       c_re(inout[0]) = tmp91 + tmp186;
	       c_re(inout[24 * iostride]) = tmp383 - tmp386;
	       c_re(inout[8 * iostride]) = tmp383 + tmp386;
	       c_im(inout[0]) = tmp435 + tmp444;
	       c_im(inout[16 * iostride]) = tmp444 - tmp435;
	       c_im(inout[8 * iostride]) = tmp445 + tmp446;
	       c_im(inout[24 * iostride]) = tmp446 - tmp445;
	  }
	  {
	       fftw_real tmp359;
	       fftw_real tmp379;
	       fftw_real tmp450;
	       fftw_real tmp452;
	       fftw_real tmp368;
	       fftw_real tmp381;
	       fftw_real tmp377;
	       fftw_real tmp380;
	       ASSERT_ALIGNED_DOUBLE();
	       {
		    fftw_real tmp351;
		    fftw_real tmp358;
		    fftw_real tmp448;
		    fftw_real tmp449;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp351 = tmp19 - tmp42;
		    tmp358 = tmp354 - tmp357;
		    tmp359 = tmp351 + tmp358;
		    tmp379 = tmp351 - tmp358;
		    tmp448 = tmp66 - tmp89;
		    tmp449 = tmp442 - tmp437;
		    tmp450 = tmp448 + tmp449;
		    tmp452 = tmp449 - tmp448;
	       }
	       {
		    fftw_real tmp360;
		    fftw_real tmp367;
		    fftw_real tmp369;
		    fftw_real tmp376;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp360 = tmp114 - tmp137;
		    tmp367 = tmp363 - tmp366;
		    tmp368 = tmp360 - tmp367;
		    tmp381 = tmp360 + tmp367;
		    tmp369 = tmp161 - tmp184;
		    tmp376 = tmp372 - tmp375;
		    tmp377 = tmp369 + tmp376;
		    tmp380 = tmp376 - tmp369;
	       }
	       {
		    fftw_real tmp378;
		    fftw_real tmp451;
		    fftw_real tmp382;
		    fftw_real tmp447;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp378 = K707106781 * (tmp368 + tmp377);
		    c_re(inout[20 * iostride]) = tmp359 - tmp378;
		    c_re(inout[4 * iostride]) = tmp359 + tmp378;
		    tmp451 = K707106781 * (tmp368 - tmp377);
		    c_im(inout[12 * iostride]) = tmp451 + tmp452;
		    c_im(inout[28 * iostride]) = tmp452 - tmp451;
		    tmp382 = K707106781 * (tmp380 - tmp381);
		    c_re(inout[28 * iostride]) = tmp379 - tmp382;
		    c_re(inout[12 * iostride]) = tmp379 + tmp382;
		    tmp447 = K707106781 * (tmp381 + tmp380);
		    c_im(inout[4 * iostride]) = tmp447 + tmp450;
		    c_im(inout[20 * iostride]) = tmp450 - tmp447;
	       }
	  }
	  {
	       fftw_real tmp391;
	       fftw_real tmp419;
	       fftw_real tmp398;
	       fftw_real tmp454;
	       fftw_real tmp422;
	       fftw_real tmp462;
	       fftw_real tmp406;
	       fftw_real tmp417;
	       fftw_real tmp457;
	       fftw_real tmp463;
	       fftw_real tmp426;
	       fftw_real tmp433;
	       fftw_real tmp413;
	       fftw_real tmp416;
	       fftw_real tmp429;
	       fftw_real tmp432;
	       ASSERT_ALIGNED_DOUBLE();
	       {
		    fftw_real tmp394;
		    fftw_real tmp397;
		    fftw_real tmp424;
		    fftw_real tmp425;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp391 = tmp387 - tmp390;
		    tmp419 = tmp387 + tmp390;
		    tmp394 = tmp392 - tmp393;
		    tmp397 = tmp395 + tmp396;
		    tmp398 = K707106781 * (tmp394 - tmp397);
		    tmp454 = K707106781 * (tmp397 + tmp394);
		    {
			 fftw_real tmp420;
			 fftw_real tmp421;
			 fftw_real tmp402;
			 fftw_real tmp405;
			 ASSERT_ALIGNED_DOUBLE();
			 tmp420 = tmp395 - tmp396;
			 tmp421 = tmp393 + tmp392;
			 tmp422 = K707106781 * (tmp420 + tmp421);
			 tmp462 = K707106781 * (tmp420 - tmp421);
			 tmp402 = tmp400 - tmp401;
			 tmp405 = tmp403 - tmp404;
			 tmp406 = (K382683432 * tmp402) - (K923879532 * tmp405);
			 tmp417 = (K923879532 * tmp402) + (K382683432 * tmp405);
		    }
		    tmp457 = tmp455 + tmp456;
		    tmp463 = tmp456 - tmp455;
		    tmp424 = tmp400 + tmp401;
		    tmp425 = tmp403 + tmp404;
		    tmp426 = (K923879532 * tmp424) - (K382683432 * tmp425);
		    tmp433 = (K382683432 * tmp424) + (K923879532 * tmp425);
		    {
			 fftw_real tmp409;
			 fftw_real tmp412;
			 fftw_real tmp427;
			 fftw_real tmp428;
			 ASSERT_ALIGNED_DOUBLE();
			 tmp409 = tmp407 - tmp408;
			 tmp412 = tmp410 - tmp411;
			 tmp413 = (K382683432 * tmp409) + (K923879532 * tmp412);
			 tmp416 = (K382683432 * tmp412) - (K923879532 * tmp409);
			 tmp427 = tmp407 + tmp408;
			 tmp428 = tmp410 + tmp411;
			 tmp429 = (K923879532 * tmp427) + (K382683432 * tmp428);
			 tmp432 = (K923879532 * tmp428) - (K382683432 * tmp427);
		    }
	       }
	       {
		    fftw_real tmp399;
		    fftw_real tmp414;
		    fftw_real tmp415;
		    fftw_real tmp418;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp399 = tmp391 - tmp398;
		    tmp414 = tmp406 - tmp413;
		    c_re(inout[30 * iostride]) = tmp399 - tmp414;
		    c_re(inout[14 * iostride]) = tmp399 + tmp414;
		    tmp415 = tmp391 + tmp398;
		    tmp418 = tmp416 + tmp417;
		    c_re(inout[22 * iostride]) = tmp415 - tmp418;
		    c_re(inout[6 * iostride]) = tmp415 + tmp418;
	       }
	       {
		    fftw_real tmp465;
		    fftw_real tmp466;
		    fftw_real tmp461;
		    fftw_real tmp464;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp465 = tmp416 - tmp417;
		    tmp466 = tmp463 - tmp462;
		    c_im(inout[14 * iostride]) = tmp465 + tmp466;
		    c_im(inout[30 * iostride]) = tmp466 - tmp465;
		    tmp461 = tmp413 + tmp406;
		    tmp464 = tmp462 + tmp463;
		    c_im(inout[6 * iostride]) = tmp461 + tmp464;
		    c_im(inout[22 * iostride]) = tmp464 - tmp461;
	       }
	       {
		    fftw_real tmp423;
		    fftw_real tmp430;
		    fftw_real tmp431;
		    fftw_real tmp434;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp423 = tmp419 - tmp422;
		    tmp430 = tmp426 - tmp429;
		    c_re(inout[26 * iostride]) = tmp423 - tmp430;
		    c_re(inout[10 * iostride]) = tmp423 + tmp430;
		    tmp431 = tmp419 + tmp422;
		    tmp434 = tmp432 + tmp433;
		    c_re(inout[18 * iostride]) = tmp431 - tmp434;
		    c_re(inout[2 * iostride]) = tmp431 + tmp434;
	       }
	       {
		    fftw_real tmp459;
		    fftw_real tmp460;
		    fftw_real tmp453;
		    fftw_real tmp458;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp459 = tmp432 - tmp433;
		    tmp460 = tmp457 - tmp454;
		    c_im(inout[10 * iostride]) = tmp459 + tmp460;
		    c_im(inout[26 * iostride]) = tmp460 - tmp459;
		    tmp453 = tmp429 + tmp426;
		    tmp458 = tmp454 + tmp457;
		    c_im(inout[2 * iostride]) = tmp453 + tmp458;
		    c_im(inout[18 * iostride]) = tmp458 - tmp453;
	       }
	  }
	  {
	       fftw_real tmp307;
	       fftw_real tmp335;
	       fftw_real tmp338;
	       fftw_real tmp492;
	       fftw_real tmp487;
	       fftw_real tmp493;
	       fftw_real tmp314;
	       fftw_real tmp484;
	       fftw_real tmp322;
	       fftw_real tmp333;
	       fftw_real tmp342;
	       fftw_real tmp349;
	       fftw_real tmp329;
	       fftw_real tmp332;
	       fftw_real tmp345;
	       fftw_real tmp348;
	       ASSERT_ALIGNED_DOUBLE();
	       {
		    fftw_real tmp306;
		    fftw_real tmp336;
		    fftw_real tmp337;
		    fftw_real tmp485;
		    fftw_real tmp310;
		    fftw_real tmp313;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp306 = K707106781 * (tmp304 - tmp305);
		    tmp307 = tmp303 - tmp306;
		    tmp335 = tmp303 + tmp306;
		    tmp336 = (K382683432 * tmp312) - (K923879532 * tmp311);
		    tmp337 = (K923879532 * tmp308) + (K382683432 * tmp309);
		    tmp338 = tmp336 + tmp337;
		    tmp492 = tmp336 - tmp337;
		    tmp485 = K707106781 * (tmp196 - tmp201);
		    tmp487 = tmp485 + tmp486;
		    tmp493 = tmp486 - tmp485;
		    tmp310 = (K382683432 * tmp308) - (K923879532 * tmp309);
		    tmp313 = (K382683432 * tmp311) + (K923879532 * tmp312);
		    tmp314 = tmp310 - tmp313;
		    tmp484 = tmp313 + tmp310;
	       }
	       {
		    fftw_real tmp318;
		    fftw_real tmp321;
		    fftw_real tmp340;
		    fftw_real tmp341;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp318 = tmp316 - tmp317;
		    tmp321 = tmp319 - tmp320;
		    tmp322 = (K195090322 * tmp318) - (K980785280 * tmp321);
		    tmp333 = (K980785280 * tmp318) + (K195090322 * tmp321);
		    tmp340 = tmp316 + tmp317;
		    tmp341 = tmp319 + tmp320;
		    tmp342 = (K831469612 * tmp340) - (K555570233 * tmp341);
		    tmp349 = (K555570233 * tmp340) + (K831469612 * tmp341);
	       }
	       {
		    fftw_real tmp325;
		    fftw_real tmp328;
		    fftw_real tmp343;
		    fftw_real tmp344;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp325 = tmp323 - tmp324;
		    tmp328 = tmp326 - tmp327;
		    tmp329 = (K195090322 * tmp325) + (K980785280 * tmp328);
		    tmp332 = (K195090322 * tmp328) - (K980785280 * tmp325);
		    tmp343 = tmp323 + tmp324;
		    tmp344 = tmp326 + tmp327;
		    tmp345 = (K831469612 * tmp343) + (K555570233 * tmp344);
		    tmp348 = (K831469612 * tmp344) - (K555570233 * tmp343);
	       }
	       {
		    fftw_real tmp315;
		    fftw_real tmp330;
		    fftw_real tmp331;
		    fftw_real tmp334;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp315 = tmp307 - tmp314;
		    tmp330 = tmp322 - tmp329;
		    c_re(inout[31 * iostride]) = tmp315 - tmp330;
		    c_re(inout[15 * iostride]) = tmp315 + tmp330;
		    tmp331 = tmp307 + tmp314;
		    tmp334 = tmp332 + tmp333;
		    c_re(inout[23 * iostride]) = tmp331 - tmp334;
		    c_re(inout[7 * iostride]) = tmp331 + tmp334;
	       }
	       {
		    fftw_real tmp495;
		    fftw_real tmp496;
		    fftw_real tmp491;
		    fftw_real tmp494;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp495 = tmp332 - tmp333;
		    tmp496 = tmp493 - tmp492;
		    c_im(inout[15 * iostride]) = tmp495 + tmp496;
		    c_im(inout[31 * iostride]) = tmp496 - tmp495;
		    tmp491 = tmp329 + tmp322;
		    tmp494 = tmp492 + tmp493;
		    c_im(inout[7 * iostride]) = tmp491 + tmp494;
		    c_im(inout[23 * iostride]) = tmp494 - tmp491;
	       }
	       {
		    fftw_real tmp339;
		    fftw_real tmp346;
		    fftw_real tmp347;
		    fftw_real tmp350;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp339 = tmp335 - tmp338;
		    tmp346 = tmp342 - tmp345;
		    c_re(inout[27 * iostride]) = tmp339 - tmp346;
		    c_re(inout[11 * iostride]) = tmp339 + tmp346;
		    tmp347 = tmp335 + tmp338;
		    tmp350 = tmp348 + tmp349;
		    c_re(inout[19 * iostride]) = tmp347 - tmp350;
		    c_re(inout[3 * iostride]) = tmp347 + tmp350;
	       }
	       {
		    fftw_real tmp489;
		    fftw_real tmp490;
		    fftw_real tmp483;
		    fftw_real tmp488;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp489 = tmp348 - tmp349;
		    tmp490 = tmp487 - tmp484;
		    c_im(inout[11 * iostride]) = tmp489 + tmp490;
		    c_im(inout[27 * iostride]) = tmp490 - tmp489;
		    tmp483 = tmp345 + tmp342;
		    tmp488 = tmp484 + tmp487;
		    c_im(inout[3 * iostride]) = tmp483 + tmp488;
		    c_im(inout[19 * iostride]) = tmp488 - tmp483;
	       }
	  }
	  {
	       fftw_real tmp203;
	       fftw_real tmp287;
	       fftw_real tmp290;
	       fftw_real tmp478;
	       fftw_real tmp473;
	       fftw_real tmp479;
	       fftw_real tmp226;
	       fftw_real tmp468;
	       fftw_real tmp254;
	       fftw_real tmp285;
	       fftw_real tmp294;
	       fftw_real tmp301;
	       fftw_real tmp281;
	       fftw_real tmp284;
	       fftw_real tmp297;
	       fftw_real tmp300;
	       ASSERT_ALIGNED_DOUBLE();
	       {
		    fftw_real tmp202;
		    fftw_real tmp288;
		    fftw_real tmp289;
		    fftw_real tmp469;
		    fftw_real tmp214;
		    fftw_real tmp225;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp202 = K707106781 * (tmp196 + tmp201);
		    tmp203 = tmp191 - tmp202;
		    tmp287 = tmp191 + tmp202;
		    tmp288 = (K923879532 * tmp224) - (K382683432 * tmp219);
		    tmp289 = (K382683432 * tmp208) + (K923879532 * tmp213);
		    tmp290 = tmp288 + tmp289;
		    tmp478 = tmp288 - tmp289;
		    tmp469 = K707106781 * (tmp305 + tmp304);
		    tmp473 = tmp469 + tmp472;
		    tmp479 = tmp472 - tmp469;
		    tmp214 = (K923879532 * tmp208) - (K382683432 * tmp213);
		    tmp225 = (K923879532 * tmp219) + (K382683432 * tmp224);
		    tmp226 = tmp214 - tmp225;
		    tmp468 = tmp225 + tmp214;
	       }
	       {
		    fftw_real tmp244;
		    fftw_real tmp253;
		    fftw_real tmp292;
		    fftw_real tmp293;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp244 = tmp232 - tmp243;
		    tmp253 = tmp249 - tmp252;
		    tmp254 = (K555570233 * tmp244) - (K831469612 * tmp253);
		    tmp285 = (K831469612 * tmp244) + (K555570233 * tmp253);
		    tmp292 = tmp232 + tmp243;
		    tmp293 = tmp249 + tmp252;
		    tmp294 = (K980785280 * tmp292) - (K195090322 * tmp293);
		    tmp301 = (K195090322 * tmp292) + (K980785280 * tmp293);
	       }
	       {
		    fftw_real tmp271;
		    fftw_real tmp280;
		    fftw_real tmp295;
		    fftw_real tmp296;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp271 = tmp259 - tmp270;
		    tmp280 = tmp276 - tmp279;
		    tmp281 = (K555570233 * tmp271) + (K831469612 * tmp280);
		    tmp284 = (K555570233 * tmp280) - (K831469612 * tmp271);
		    tmp295 = tmp259 + tmp270;
		    tmp296 = tmp276 + tmp279;
		    tmp297 = (K980785280 * tmp295) + (K195090322 * tmp296);
		    tmp300 = (K980785280 * tmp296) - (K195090322 * tmp295);
	       }
	       {
		    fftw_real tmp227;
		    fftw_real tmp282;
		    fftw_real tmp283;
		    fftw_real tmp286;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp227 = tmp203 + tmp226;
		    tmp282 = tmp254 + tmp281;
		    c_re(inout[21 * iostride]) = tmp227 - tmp282;
		    c_re(inout[5 * iostride]) = tmp227 + tmp282;
		    tmp283 = tmp203 - tmp226;
		    tmp286 = tmp284 - tmp285;
		    c_re(inout[29 * iostride]) = tmp283 - tmp286;
		    c_re(inout[13 * iostride]) = tmp283 + tmp286;
	       }
	       {
		    fftw_real tmp477;
		    fftw_real tmp480;
		    fftw_real tmp481;
		    fftw_real tmp482;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp477 = tmp285 + tmp284;
		    tmp480 = tmp478 + tmp479;
		    c_im(inout[5 * iostride]) = tmp477 + tmp480;
		    c_im(inout[21 * iostride]) = tmp480 - tmp477;
		    tmp481 = tmp254 - tmp281;
		    tmp482 = tmp479 - tmp478;
		    c_im(inout[13 * iostride]) = tmp481 + tmp482;
		    c_im(inout[29 * iostride]) = tmp482 - tmp481;
	       }
	       {
		    fftw_real tmp291;
		    fftw_real tmp298;
		    fftw_real tmp299;
		    fftw_real tmp302;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp291 = tmp287 + tmp290;
		    tmp298 = tmp294 + tmp297;
		    c_re(inout[17 * iostride]) = tmp291 - tmp298;
		    c_re(inout[iostride]) = tmp291 + tmp298;
		    tmp299 = tmp287 - tmp290;
		    tmp302 = tmp300 - tmp301;
		    c_re(inout[25 * iostride]) = tmp299 - tmp302;
		    c_re(inout[9 * iostride]) = tmp299 + tmp302;
	       }
	       {
		    fftw_real tmp467;
		    fftw_real tmp474;
		    fftw_real tmp475;
		    fftw_real tmp476;
		    ASSERT_ALIGNED_DOUBLE();
		    tmp467 = tmp301 + tmp300;
		    tmp474 = tmp468 + tmp473;
		    c_im(inout[iostride]) = tmp467 + tmp474;
		    c_im(inout[17 * iostride]) = tmp474 - tmp467;
		    tmp475 = tmp294 - tmp297;
		    tmp476 = tmp473 - tmp468;
		    c_im(inout[9 * iostride]) = tmp475 + tmp476;
		    c_im(inout[25 * iostride]) = tmp476 - tmp475;
	       }
	  }
     }
}

static const int twiddle_order[] =
{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31};
fftw_codelet_desc fftwi_twiddle_32_desc =
{
     "fftwi_twiddle_32",
     (void (*)()) fftwi_twiddle_32,
     32,
     FFTW_BACKWARD,
     FFTW_TWIDDLE,
     520,
     31,
     twiddle_order,
};


/**
 * @file utilities.h
 * @author Matthew Nicely (mnicely@nvidia.com)
 * @date 2020-01-06
 * @version 1.0
 * @brief Contains additional functions used throughout
 * the application.
 *
 * @copyright Copyright (c) 2020
 *
 * @license This project is released under the GNU Public License
 *
 * @note * Target Processor: Intel x86
 * @n * Target Compiler: GCC 7.4.0
 * @n * NVCC Compiler: CUDA Toolkit 10.0 or later
 */
#ifndef UTILITIES_H_
#define UTILITIES_H_

#include <algorithm>  // std::remove_if, std::nth_element
#include <cassert>    // assert
#include <cblas.h>    //cblas_sgemm
#include <cmath>      // std::sqrt,
#include <fstream>    // std::ofstream
#include <iterator>   // std::vector<>::const_iterator
#include <lapacke.h>  // LAPACKE_sgetrf, LAPACKE_sgetri
#include <map>        // std::map
#include <numeric>    // std::accumulate, std::inner_product
#include <random>     // std::default_random_engine, std::random_device, std::uniform_real_distribution
#include <stdexcept>  // std::runtime_error
#include <string>     // std::string
#include <vector>     // std::vector

// *************** FOR ERROR CHECKING *******************
#ifndef CUDA_RT_CALL
#define CUDA_RT_CALL( call )                                                                                \
    {                                                                                \
        auto status = static_cast<cudaError_t>( call );                                                                \
        if ( status != cudaSuccess )                                                                                \
            fprintf( stderr,                                                                                \
                     "ERROR: CUDA RT call \"%s\" in line %d of file %s failed "                                        \
                     "with "                                                                                \
                     "%s (%d).\n",                                                                                \
                     #call,                                                                                \
                     __LINE__,                                                                                \
                     __FILE__,                                                                                \
                     cudaGetErrorString( status ),                                                                     \
                     status );                                                                                \
    }
#endif  // CUDA_RT_CALL
// *************** FOR ERROR CHECKING *******************

/**
 * @brief Contains additional functions used throughout application
 */
namespace utility {

constexpr int kSysDim { 4 };  /**< Number of states in system model */
constexpr int kMeasDim { 2 }; /**< Number of states in measurement model */

/**
 * @enum Processor
 * @brief Enumeration of processor types
 */
enum class Processor : bool { kCpu, kGpu };

using ProcessorMap = std::map<Processor, std::string>;

static const ProcessorMap processor_map = { { Processor::kCpu, "CPU" }, { Processor::kGpu, "GPU" } };

/**
 * @enum Timing
 * @brief Enumeration of timing calculations
 */
enum class Timing : int { kMedian, kMean, kStdDev, kCount };

using TimingMap = std::map<Timing, std::string>;

static const TimingMap timing_map = { { Timing::kMedian, "Median" },
                                      { Timing::kMean, "Mean" },
                                      { Timing::kStdDev, "StdDev" } };

/**
 * @enum Filter
 * @brief Enumeration of possible particle filters
 */
enum class Filter : int { kBootstrap, kCount };

using FilterMap = std::map<Filter, std::string>;

static const FilterMap filters_map = { { Filter::kBootstrap, "Bootstrap" } };

/**
 * @enum Method
 * @brief Enumeration of possible resampling method
 */
enum class Method : int { kSystematic, kStratified, kMetropolisC2, kCount };

using MethodMap = std::map<Method, std::string>;

static const MethodMap method_map = { { Method::kSystematic, "Systematic" },
                                      { Method::kStratified, "Stratified" },
                                      { Method::kMetropolisC2, "MetropolisC2" } };

/**
 * @struct FilterInfo
 * @brief Structure to store user-specified data and send to filter classes
 */
typedef struct _filterInfo {
    int samples;     /**< Quantity of samples specified from input file */
    int particles;   /**< Quantity of particles specified from input file */
    int num_mcs;     /**< Quantity of Monte Carlos specified from input file */
    int resampling;  /**< Systmatic = 0, Stratified = 1, Metropolis = 2 */
    int filter_type; /**< Type of filter: Bootstrap = 0 */
} FilterInfo;

/**
 * @struct Matrix
 * @brief Structure to store matrix and information
 *
 * @param[in] row Stores quantity of rows
 * @param[in] col Stores quantity of columns
 * @param[in] val Store matrix data in a vector
 */
typedef struct _matrix {

    int                row;
    int                col;
    std::vector<float> val;

    _matrix( const _matrix &a ) : row( a.row ), col( a.col ), val( a.val ) {};

    _matrix( int const &row, int const &col, std::initializer_list<float> const &list ) :
        row( row ), col( col ), val( list ) {}

    _matrix( int const &row, int const &col ) : row( row ), col( col ) {
        val.resize( row * col, 0.0f );
    }

    _matrix( int const &row, int const &col, std::vector<float> const val ) : row( row ), col( col ), val( val ) {}
} Matrix;

/**
 * @brief Template function to compute square root of a matrix using floats
 *
 * @see http://www.netlib.org/lapack/lapacke.html
 * @see
 * http://www.netlib.org/lapack/explore-html/d7/d75/lapacke__ssyevr_8c_source.html
 * @see http://www.openblas.net/
 * @see http://www.seehuhn.de/pages/matrixfn
 *
 * @param[in,out] matrix_a Matrix to process
 */
inline void MatrixSqrt( utility::Matrix &matrix_a ) {

    int    n { matrix_a.row };
    float *a { matrix_a.val.data( ) };

    /* allocate space for the output parameters and workspace arrays */
    int   m {};
    float abstol { -1.0f };

    std::vector<float> w( n, 0.0f );
    std::vector<float> z( n * n, 0.0f );
    std::vector<int>   suppz( 2 * n, 0 );

    /* get the eigenvalues and eigenvectors */
    int info = LAPACKE_ssyevr(
        LAPACK_ROW_MAJOR, 'V', 'A', 'L', n, a, n, 0, 0, 0, 0, abstol, &m, w.data( ), z.data( ), n, suppz.data( ) );

    if ( info != 0 ) {
        throw std::runtime_error( "Issue with LAPACKE_ssyevr.\n" );
    }

    /* allocate and Initialize a new matrix B=Z*D */
    std::vector<float> b( n * n, 0.0f );
    for ( int j = 0; j < n; ++j ) {
        float lambda = std::sqrt( w[j] );
        for ( int i = 0; i < n; ++i ) {
            b[i * n + j] = z[i * n + j] * lambda;
        }
    }

    float alpha { 1.0f };
    float beta { 0.0f };

    /* calculate the square root A=B*Z^T */
    cblas_sgemm( CblasRowMajor, CblasNoTrans, CblasTrans, n, n, n, alpha, b.data( ), n, z.data( ), n, beta, a, n );
}

/**
 * @brief Matrix-Matrix multiplication function using floats
 *
 * @see http://www.openblas.net/
 *
 * @param[out] matrix_c Structure to store A * B
 * @param[in] matrix_a Structure reference to matrix A
 * @param[in] matrix_b Structure reference to matrix B
 */

inline void MatrixMult( utility::Matrix &matrix_c, utility::Matrix const &matrix_a, utility::Matrix const &matrix_b ) {

    assert( matrix_a.col == matrix_b.row );

    float const *a { matrix_a.val.data( ) };
    float const *b { matrix_b.val.data( ) };
    float *      c { matrix_c.val.data( ) };

    float alpha { 1.0f };
    float beta { 0.0f };

    cblas_sgemm( CblasRowMajor,
                 CblasNoTrans,
                 CblasNoTrans,
                 matrix_a.row,
                 matrix_b.col,
                 matrix_b.row,
                 alpha,
                 a,
                 matrix_b.row,
                 b,
                 matrix_b.col,
                 beta,
                 c,
                 matrix_b.col );
}

/**
 * @brief Matrix-Matrix-Matrix multiplication function using floats
 *
 * @see http://www.openblas.net/
 *
 * @param[out] matrix_d Structure to store A * B * C
 * @param[in] matrix_a Structure reference to matrix A
 * @param[in] matrix_b Structure reference to matrix B
 * @param[in] matrix_c Structure reference to matrix C
 */
inline void MatrixMult( utility::Matrix &      matrix_d,
                        utility::Matrix const &matrix_a,
                        utility::Matrix const &matrix_b,
                        utility::Matrix const &matrix_c ) {

    assert( matrix_a.col == matrix_b.row );
    assert( matrix_b.col == matrix_c.row );

    float const *a { matrix_a.val.data( ) };
    float const *b { matrix_b.val.data( ) };
    float const *c { matrix_c.val.data( ) };
    float *      d { matrix_d.val.data( ) };

    std::vector<float> temp_storage( matrix_a.row * matrix_b.col, 0.0f );
    float *            ab { temp_storage.data( ) };  // Temp store for A*B

    float alpha { 1.0f };
    float beta { 0.0f };

    cblas_sgemm( CblasRowMajor,
                 CblasNoTrans,
                 CblasNoTrans,
                 matrix_a.row,
                 matrix_b.col,
                 matrix_b.row,
                 alpha,
                 a,
                 matrix_b.row,
                 b,
                 matrix_b.col,
                 beta,
                 ab,
                 matrix_b.col );

    cblas_sgemm( CblasRowMajor,
                 CblasNoTrans,
                 CblasNoTrans,
                 matrix_a.row,
                 matrix_c.col,
                 matrix_c.row,
                 alpha,
                 ab,
                 matrix_c.row,
                 c,
                 matrix_c.col,
                 beta,
                 d,
                 matrix_c.col );
}

/**
 * @brief Matrix-Vector multiplication function using floats
 *
 * @see http://www.openblas.net/
 *
 * @param[out] vector_c Vector to store A * B
 * @param[in] matrix_a Structure reference to matrix A
 * @param[in] vector_b Vector reference to vector B
 */
inline void
MatrixMult( std::vector<float> &vector_c, utility::Matrix const &matrix_a, std::vector<float> const &vector_b ) {

    assert( matrix_a.col == vector_b.size( ) );

    float const *a { matrix_a.val.data( ) };
    float const *b { vector_b.data( ) };
    float *      c { vector_c.data( ) };

    float alpha { 1.0f };
    float beta { 0.0f };

    cblas_sgemv( CblasRowMajor, CblasNoTrans, matrix_a.row, matrix_a.col, alpha, a, matrix_a.col, b, 1, beta, c, 1 );
}

/**
 * @brief Function to compute the inversion of a matrix using floats
 *
 * @see http://www.netlib.org/lapack/lapacke.html
 * @see
 * http://www.netlib.org/lapack/explore-html/d2/d96/lapacke__dgetrf_8c_source.html
 * @see
 * http://www.netlib.org/lapack/explore-html/da/d0e/lapacke__dgetri_8c_source.html
 * @see
 * https://stackoverflow.com/questions/3519959/computing-the-inverse-of-a-matrix-using-lapack-in-c
 *
 * @param[in,out] matrix_a Matrix to be inverted
 */
inline void ComputeInverse( utility::Matrix &matrix_a ) {

    // A must be a square matrix???
    int              n { matrix_a.row };
    float *          a = matrix_a.val.data( );
    std::vector<int> piv( n + 1, 0 );

    int info {};
    info = LAPACKE_sgetrf( LAPACK_ROW_MAJOR, n, n, a, n, piv.data( ) );

    if ( info != 0 ) {
        throw std::runtime_error( "Issue with LAPACKE_sgetrf.\n" );
    }

    info = LAPACKE_sgetri( LAPACK_ROW_MAJOR, n, a, n, piv.data( ) );

    if ( info != 0 ) {
        throw std::runtime_error( "Issue with LAPACKE_sgetri.\n" );
    }
}

/**
 * @brief Normal distribution random number generator
 *
 * @param[in] random_numbers Stores random numbers
 * @param[in] lBound Lower bound of distribution
 * @param[in] uBound Upper bound of distribution
 */
inline void GenerateRandomNum( std::vector<float> &random_numbers, float const &lBound, float const &uBound ) {

    // Random device class instance, source of 'true' randomness for
    // initializing random seed
    std::default_random_engine gen( std::random_device {}( ) );

    // Instance of class std::normal_distribution with specific mean and stddev
    // Lower and Upper limits are declared in base class
    std::normal_distribution<float> distr( lBound, uBound );

    // Compute random numbers for all particles
    for_each( random_numbers.begin( ), random_numbers.end( ), [&distr, &gen]( float &a ) { a = distr( gen ); } );
}

/**
 * @brief Compute median for Monte Carlos times
 *
 * @param[in/out] time_vector Vector holding times for all Monte Carlos
 * @param[out] median Median time for all Monte Carlos
 */
inline void ComputeMedianTime( std::vector<float> &time_vector, float &median ) {
    // Compute median
    std::nth_element( time_vector.begin( ), time_vector.begin( ) + time_vector.size( ) / 2, time_vector.end( ) );
    median = time_vector[time_vector.size( ) / 2];

    // Remove outliers
    time_vector.erase( std::remove_if( time_vector.begin( ),
                                       time_vector.end( ),
                                       [&median]( float const &n ) { return ( n > ( median * 1.05f ) ); } ),
                       time_vector.end( ) );
}

/**
 * @brief Compute mean for Monte Carlos times
 *
 * @param[in] time_vector Vector holding times for all Monte Carlos
 * @param[out] mean Mean time for all Monte Carlos
 */
inline void ComputeMeanTime( std::vector<float> const &time_vector, float &mean ) {
    // Compute mean
    mean = std::accumulate( time_vector.begin( ), time_vector.end( ), 0.0f ) / time_vector.size( );
}

/**
 * @brief Compute standard deviation for Monte Carlos times
 *
 * @param[in] time_vector Vector holding times for all Monte Carlos
 * @param[in] mean Mean time for all Monte Carlos
 * @param[out] stdDev
 */
inline void ComputeStdDevTime( std::vector<float> const &time_vector, float const &mean, float &stdDev ) {

    // Compute standard deviation
    std::vector<float> diff( time_vector.size( ) );
    std::transform(
        time_vector.begin( ), time_vector.end( ), diff.begin( ), [&mean]( float const &x ) { return ( x - mean ); } );

    // Compute difference
    float sqrtSum { std::inner_product( diff.begin( ), diff.end( ), diff.begin( ), 0.0f ) };  // Compute product
    stdDev = std::sqrt( sqrtSum / time_vector.size( ) );
}

/**
 * @brief Compute median, mean, standard deviation for Monte Carlo times
 *
 * @param[in/out] time_vector
 * @param[in/out] median
 * @param[in/out] mean
 * @param[in/out] stdDev
 */
inline void ComputeOverallTiming( std::vector<float> &timeVec, float &median, float &mean, float &stdDev ) {
    ComputeMedianTime( timeVec, median );
    ComputeMeanTime( timeVec, mean );
    ComputeStdDevTime( timeVec, mean, stdDev );
}

/**
 * @brief Compute overall timing across all Monte Carlos
 *
 * @param[in] timing_results 2D vector containing timing results for each Monte
 * Carlo
 */
inline void PrintTimingResults( std::vector<std::vector<float>> const &timing_results ) {

    for ( int i = 0; i < static_cast<int>( utility::Timing::kCount ); i++ ) {
        std::printf( "%s\t ", utility::timing_map.find( utility::Timing( i ) )->second.c_str( ) );
    }
    std::printf( "\n" );

    for ( int i = 0; i < static_cast<int>( utility::Timing::kCount ); i++ ) {
        float sum { std::accumulate( timing_results[i].begin( ), timing_results[i].end( ), 0.0f ) };
        float mean { sum / timing_results[i].size( ) };
        std::printf( "%0.0f\t ", mean );
    }
    std::printf( "\n\n" );
}

/**
 * @brief Writes data to output file for generated system
 *        and measurement truth data
 *
 * @param[in] filename Output filename
 * @param[in] dim Number of system dimensions
 * @param[in] samples Number of samples
 * @param[in] it Iterator to beginning of vector to be printed
 */
inline void WriteToFile( std::string const &                          filename,
                         int const &                                  dim,
                         int const &                                  samples,
                         typename std::vector<float>::const_iterator &it ) {

    // We jump to the end to add each Monte Carlo
    std::ofstream fout( filename, std::ios_base::app );
    fout.precision( 15 );
    fout.setf( std::ios::fixed, std::ios::floatfield );
    // fout << dim << " " << samples << "\n";
    for ( int i = 0; i < samples; i++ ) {
        int idxS { i * dim };
        int idxE { idxS + dim };
        std::copy( it + idxS, it + idxE, std::ostream_iterator<float>( fout, " " ) );
        fout << "\n";
    }
    fout.close( );
}

/**
 * @brief Writes data to output file for filtered estimates
 *
 * @param[in] filename Output filename
 * @param[in] type Filter type
 * @param[in] samples Number of samples
 * @param[in] median Timing median across all Monte Carlos
 * @param[in] mean Timing mean across all Monte Carlos
 * @param[in] stdDev Timing standard deviation across all Monte Carlos
 * @param[in] it Iterator to beginning of vector to be printed
 */
inline void WriteToFile( std::string const &                          filename,
                         std::string const &                          type,
                         int const &                                  samples,
                         float const &                                median,
                         float const &                                mean,
                         float const &                                stdDev,
                         typename std::vector<float>::const_iterator &it ) {

    // We jump to the end to add each Monte Carlo
    std::ofstream fout( filename, std::ios_base::app );
    fout.setf( std::ios::fixed, std::ios::floatfield );
    fout << type << " " << median << " " << mean << " " << stdDev << "\n";
    fout.precision( 15 );
    for ( int i = 0; i < samples; i++ ) {
        int idxS { i * utility::kSysDim };
        int idxE { idxS + utility::kSysDim };
        std::copy( it + idxS, it + idxE, std::ostream_iterator<float>( fout, " " ) );
        fout << "\n";
    }
    fout.close( );
}

/**
 * @brief Write header to output file for generated system
 *        and measurement truth data
 *
 * @param[in] filename Output filename
 * @param[in] info Struct containing filter information
 */
inline void WriteTruthHeader( std::string const &filename, int const &num_mcs, int const &dim, int const &samples ) {
    std::ofstream fout( filename, std::ios_base::trunc );
    fout.setf( std::ios::fixed, std::ios::floatfield );
    fout << num_mcs << " " << dim << " " << samples << "\n";
    fout.close( );
}

/**
 * @brief Write header to output file.
 *
 * @param[in] filename Output filename
 * @param[in] info Struct containing filter information
 */
inline void WriteDataHeader( std::string const &filename, utility::FilterInfo const &filter_info ) {
    std::ofstream fout( filename, std::ios_base::trunc );
    fout.setf( std::ios::fixed, std::ios::floatfield );
    fout << filter_info.num_mcs << " " << kSysDim << " " << filter_info.samples << " " << filter_info.particles << "\n";
    fout.close( );
}

#ifdef USE_NVTX
#include "nvtx3/nvToolsExt.h"

uint32_t const colors[] { 0x0000ff00, 0x0f0000ff, 0x000000ff, 0x00ffff00, 0x00ff00ff,
                          0x0000ffff, 0x00ff0000, 0x00ffffff, 0x0f0f0f0f };
int const      num_colors { sizeof( colors ) / sizeof( uint32_t ) };

/**
 * class Tracer
 */
class Tracer {
  public:
    /**
     * @brief Construct a new Tracer object
     *
     * @see
     * https://docs.nvidia.com/gameworks/content/gameworkslibrary/nvtx/nvidia_tools_extension_library_nvtx.htm
     * @see
     * https://devblogs.nvidia.com/cuda-pro-tip-generate-custom-application-profile-timelines-nvtx/
     *
     * @param[in] name Function name
     * @param[in] cid color id
     */

    Tracer( char const *name, int const &cid ) {
        int color_id                      = cid;
        color_id                          = color_id % num_colors;
        nvtxEventAttributes_t eventAttrib = {};
        eventAttrib.version               = NVTX_VERSION;
        eventAttrib.size                  = NVTX_EVENT_ATTRIB_STRUCT_SIZE;
        eventAttrib.colorType             = NVTX_COLOR_ARGB;
        eventAttrib.color                 = colors[color_id];
        eventAttrib.messageType           = NVTX_MESSAGE_TYPE_ASCII;
        eventAttrib.message.ascii         = name;
        nvtxRangePushEx( &eventAttrib );
    }

    /**
     * @brief Destroy the Tracer object
     *
     */
    ~Tracer( ) {
        nvtxRangePop( );
    }
};

#define RANGE( name, cid ) utility::Tracer uniq_name_using_macros( name, cid );
#else
#define RANGE( name, cid )
#endif

} /* namespace utility */

#endif /* UTILITIES_H_ */