FilteredBFS.cpp

Go to the documentation of this file.

/****************************************************************/
/* Parallel Combinatorial BLAS Library (for Graph Computations) */
/* version 1.6 -------------------------------------------------*/
/* date: 6/15/2017 ---------------------------------------------*/
/* authors: Ariful Azad, Aydin Buluc  --------------------------*/
/****************************************************************/
/*
 Copyright (c) 2010-2017, The Regents of the University of California
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
 in the Software without restriction, including without limitation the rights
 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 copies of the Software, and to permit persons to whom the Software is
 furnished to do so, subject to the following conditions:
 
 The above copyright notice and this permission notice shall be included in
 all copies or substantial portions of the Software.
 
 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 */
 
#define DETERMINISTIC
#include "CombBLAS/CombBLAS.h"
#include <mpi.h>
#include <sys/time.h> 
#include <iostream>
#include <functional>
#include <algorithm>
#include <vector>
#include <string>
#include <sstream>
#ifdef THREADED
    #ifndef _OPENMP
    #define _OPENMP
    #endif
    #include <omp.h>
#endif
 
#ifdef USE_PAPI
    #include <papi.h>
    #include "papi_combblas_global.h"
#endif
 
 
/* Global variables for timing */
 
double cblas_alltoalltime;
double cblas_allgathertime;
int cblas_splits;
 
/* End global variables */
 
 
#include "TwitterEdge.h"
 
#define MAX_ITERS 20000
#define EDGEFACTOR 16
#define ITERS 16 
#define CC_LIMIT 100
#define PERCENTS 4  // testing with 4 different percentiles
#define MINRUNS 4
//#define ONLYTIME // don't calculate TEPS
 
using namespace std;
using namespace combblas;
 
 
template <typename PARMAT>
void Symmetricize(PARMAT & A)
{
    // boolean addition is practically a "logical or"
    // therefore this doesn't destruct any links
    PARMAT AT = A;
    AT.Transpose();
    A += AT;
}
 
 
#ifdef DETERMINISTIC
MTRand GlobalMT(1);
#else
MTRand GlobalMT;    // generate random numbers with Mersenne Twister 
#endif
 
 
struct Twitter_obj_randomizer : public std::unary_function<TwitterEdge, TwitterEdge>
{
  const TwitterEdge operator()(const TwitterEdge & x) const
  {
    short mycount = 1;
    bool myfollow = 0;
    time_t mylatest = static_cast<int64_t>(GlobalMT.rand() * 10000);    // random.randrange(0,10000)
 
    return TwitterEdge(mycount, myfollow, mylatest);
  }
};
 
struct Twitter_materialize: public std::binary_function<TwitterEdge, time_t, bool>
{
    bool operator()(const TwitterEdge & x, time_t sincedate) const
    {
        if(x.isRetwitter() && x.LastTweetBy(sincedate)) 
            return false;   // false if the edge is going to be kept
        else
            return true;    // true if the edge is to be pruned
    }
};
 
#ifdef USE_PAPI
void CheckPAPI(int errorcode, char [] errorstring)
{
    if (errorcode != PAPI_OK) 
    {
        PAPI_perror(errorcode, errorstring, PAPI_MAX_STR_LEN);
        fprintf(stderr, "PAPI error (%d): %s\n", errorcode, errorstring);
    }
}
#endif
 
 
 
int main(int argc, char* argv[])
{
    int nprocs, myrank; 
#ifdef _OPENMP
    int cblas_splits = omp_get_max_threads();
        int provided, flag, claimed;
        MPI_Init_thread(&argc, &argv, MPI_THREAD_FUNNELED, &provided );
        MPI_Is_thread_main( &flag );
        if (!flag)
            SpParHelper::Print("This thread called init_thread but Is_thread_main gave false\n");
        MPI_Query_thread( &claimed );
        if (claimed != provided)
            SpParHelper::Print("Query thread gave different thread level than requested\n");
#else
    MPI_Init(&argc, &argv);
    int cblas_splits = 1;   
#endif
 
    MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
    MPI_Comm_rank(MPI_COMM_WORLD,&myrank);
    int MAXTRIALS;
    int retval;
 
#ifdef USE_PAPI
    /* Initialize the PAPI library */
    retval = PAPI_library_init(PAPI_VER_CURRENT);
    if (retval != PAPI_VER_CURRENT && retval > 0) 
    {
        fprintf(stderr,"PAPI library version mismatch!\en");
        exit(1); 
    }
    retval = PAPI_is_initialized();
    if (retval != PAPI_LOW_LEVEL_INITED)
        cout << "Not initialized" << endl;
#endif
 
    if(argc < 3)
    {
        if(myrank == 0)
        {
            cout << "Usage: ./FilteredBFS <File, Gen> <Input Name | Scale> (Optional: Double)" << endl;
            cout << "Example: ./FilteredBFS File twitter_small.txt Double" << endl;
        }
        MPI_Finalize();
        return -1;
    }       
    {
        typedef SpParMat < int64_t, TwitterEdge, SpDCCols<int64_t, TwitterEdge > > PSpMat_Twitter;
        typedef SpParMat < int64_t, bool, SpDCCols<int64_t, bool > > PSpMat_Bool;
        shared_ptr<CommGrid> fullWorld;
        fullWorld.reset( new CommGrid(MPI_COMM_WORLD, 0, 0) );
 
        // Declare objects
        PSpMat_Twitter A(fullWorld);
        FullyDistVec<int64_t, int64_t> indegrees(fullWorld);    // in-degrees of vertices (including multi-edges and self-loops)
        FullyDistVec<int64_t, int64_t> oudegrees(fullWorld);    // out-degrees of vertices (including multi-edges and self-loops)
        FullyDistVec<int64_t, int64_t> degrees(fullWorld);  // combined degrees of vertices (including multi-edges and self-loops)
        PSpMat_Bool * ABool;
 
        double t01 = MPI_Wtime();
        if(string(argv[1]) == string("File")) // text|binary input option
        {
            SpParHelper::Print("Using real data, which we NEVER permute for load balance, also leaving isolated vertices as-is, if any\n");
            SpParHelper::Print("This is because the input is assumed to be load balanced\n");
            SpParHelper::Print("BFS is run on DIRECTED graph, hence hitting SCCs, and TEPS is unidirectional\n");
 
            // ReadDistribute (const string & filename, int master, bool nonum, HANDLER handler, bool transpose, bool pario)
            // if nonum is true, then numerics are not supplied and they are assumed to be all 1's
            A.ReadDistribute(string(argv[2]), 0, false, TwitterReadSaveHandler<int64_t>(), true, true); // read it from file (and transpose on the fly)
 
            A.PrintInfo();
            SpParHelper::Print("Read input\n");
 
            ABool = new PSpMat_Bool(A);
            if(argc == 4 && string(argv[3]) == string("Double"))
            {
                MAXTRIALS = 2;
            }
            else 
            {
                MAXTRIALS = 1;
            }
        }
        else if(string(argv[1]) == string("Gen"))
        {
            SpParHelper::Print("Using synthetic data, which we ALWAYS permute for load balance\n");
            SpParHelper::Print("We only balance the original input, we don't repermute after each filter change\n");
            SpParHelper::Print("BFS is run on UNDIRECTED graph, hence hitting CCs, and TEPS is bidirectional\n");
 
            double initiator[4] = {.57, .19, .19, .05};
            double t01 = MPI_Wtime();
            double t02;
            DistEdgeList<int64_t> * DEL = new DistEdgeList<int64_t>();
 
            unsigned scale = static_cast<unsigned>(atoi(argv[2]));
            ostringstream outs;
            outs << "Forcing scale to : " << scale << endl;
            SpParHelper::Print(outs.str());
 
            // parameters: (double initiator[4], int log_numverts, int edgefactor, bool scramble, bool packed)
            DEL->GenGraph500Data(initiator, scale, EDGEFACTOR, true, true );    // generate packed edges
            SpParHelper::Print("Generated renamed edge lists\n");
 
            ABool = new PSpMat_Bool(*DEL, false); 
            int64_t removed  = ABool->RemoveLoops();
            ostringstream loopinfo;
            loopinfo << "Converted to Boolean and removed " << removed << " loops" << endl;
            SpParHelper::Print(loopinfo.str());
            ABool->PrintInfo();
            delete DEL; // free memory
            A = PSpMat_Twitter(*ABool); // any upcasting generates the default object
            
            MAXTRIALS = PERCENTS;   // benchmarking
        }
        else 
        {   
            SpParHelper::Print("Not supported yet\n");
            return 0;
        }
        double t02 = MPI_Wtime();           
        ostringstream tinfo;
        tinfo << "I/O (or generation) took " << t02-t01 << " seconds" << endl;                
        SpParHelper::Print(tinfo.str());
 
        // indegrees is sum along rows because A is loaded as "tranposed", similarly oudegrees is sum along columns
        ABool->PrintInfo();
        ABool->Reduce(oudegrees, Column, plus<int64_t>(), static_cast<int64_t>(0));     
        ABool->Reduce(indegrees, Row, plus<int64_t>(), static_cast<int64_t>(0));    
        
        // indegrees_filt and oudegrees_filt is used for the real data
        FullyDistVec<int64_t, int64_t> indegrees_filt(fullWorld);
        FullyDistVec<int64_t, int64_t> oudegrees_filt(fullWorld);
 
        typedef FullyDistVec<int64_t, int64_t> IntVec;  // used for the synthetic data (symmetricized before randomization)
            FullyDistVec<int64_t, int64_t> degrees_filt[4] = {IntVec(fullWorld), IntVec(fullWorld), IntVec(fullWorld), IntVec(fullWorld)};  
        int64_t keep[PERCENTS] = {100, 1000, 2500, 10000};  // ratio of edges kept in range (0, 10000) 
        
        if(string(argv[1]) == string("File"))   // if using synthetic data, no notion of out/in degrees after randomization exist
        {
            struct tm timeinfo;
            memset(&timeinfo, 0, sizeof(struct tm));
            int year, month, day, hour, min, sec;
            year = 2009;    month = 7;  day = 1;
            hour = 0;   min = 0;    sec = 0;
            
            timeinfo.tm_year = year - 1900; // year is "years since 1900"
            timeinfo.tm_mon = month - 1 ;   // month is in range 0...11
            timeinfo.tm_mday = day;         // range 1...31
            timeinfo.tm_hour = hour;        // range 0...23
            timeinfo.tm_min = min;          // range 0...59
            timeinfo.tm_sec = sec;          // range 0.
            time_t mysincedate = timegm(&timeinfo);
            
            PSpMat_Twitter B = A;
            B.Prune(bind2nd(Twitter_materialize(), mysincedate));
            PSpMat_Bool BBool = B;
            BBool.PrintInfo();
            BBool.Reduce(oudegrees_filt, Column, plus<int64_t>(), static_cast<int64_t>(0));     
            BBool.Reduce(indegrees_filt, Row, plus<int64_t>(), static_cast<int64_t>(0)); 
        }
 
        degrees = indegrees;    
        degrees.EWiseApply(oudegrees, plus<int64_t>());
        SpParHelper::Print("All degrees calculated\n");
        delete ABool;
 
        float balance = A.LoadImbalance();
        ostringstream outs;
        outs << "Load balance: " << balance << endl;
        SpParHelper::Print(outs.str());
 
        if(string(argv[1]) == string("Gen"))
        {
            // We symmetricize before we apply the random generator
            // Otherwise += will naturally add the random numbers together
            // hence will create artificially high-permeable filters
            Symmetricize(A);    // A += A';
            SpParHelper::Print("Symmetricized\n");  
 
            A.Apply(Twitter_obj_randomizer());
            A.PrintInfo();
            
            FullyDistVec<int64_t, int64_t> * nonisov = new FullyDistVec<int64_t, int64_t>(degrees.FindInds(bind2nd(greater<int64_t>(), 0)));    
            SpParHelper::Print("Found (and permuted) non-isolated vertices\n"); 
            nonisov->RandPerm();    // so that A(v,v) is load-balanced (both memory and time wise)
            A(*nonisov, *nonisov, true);    // in-place permute to save memory
            SpParHelper::Print("Dropped isolated vertices from input\n");   
            
            indegrees = indegrees(*nonisov);    // fix the degrees arrays too
            oudegrees = oudegrees(*nonisov);    
            degrees = degrees(*nonisov);
            delete nonisov;
            
            for (int i=0; i < PERCENTS; i++) 
            {
                PSpMat_Twitter B = A;
                B.Prune(bind2nd(Twitter_materialize(), keep[i]));
                PSpMat_Bool BBool = B;
                BBool.PrintInfo();
                float balance = B.LoadImbalance();
                ostringstream outs;
                outs << "Load balance of " << static_cast<float>(keep[i])/100 << "% filtered case: " << balance << endl;
                SpParHelper::Print(outs.str());
                
                // degrees_filt[i] is by-default generated as permuted
                BBool.Reduce(degrees_filt[i], Column, plus<int64_t>(), static_cast<int64_t>(0));  // Column=Row since BBool is symmetric
            }
        }
        // real data is pre-balanced, because otherwise even the load balancing routine crushes due to load imbalance
            
        float balance_former = A.LoadImbalance();
        ostringstream outs_former;
        outs_former << "Load balance: " << balance_former << endl;
        SpParHelper::Print(outs_former.str());
 
        MPI_Barrier(MPI_COMM_WORLD);
        double t1 = MPI_Wtime();
 
        FullyDistVec<int64_t, int64_t> Cands(MAX_ITERS, 0);
        double nver = (double) degrees.TotalLength();
        Cands.SelectCandidates(nver);
 
        for(int trials =0; trials < MAXTRIALS; trials++)    
        {
            if(string(argv[1]) == string("Gen"))
            {
                LatestRetwitterBFS::sincedate = keep[trials];
                ostringstream outs;
                outs << "Initializing since date (only once) to " << LatestRetwitterBFS::sincedate << endl;
                SpParHelper::Print(outs.str());
            }
 
            cblas_allgathertime = 0;
            cblas_alltoalltime = 0;
 
            double MTEPS[ITERS]; double INVMTEPS[ITERS]; double TIMES[ITERS]; double EDGES[ITERS];
            double MPEPS[ITERS]; double INVMPEPS[ITERS];
            int sruns = 0;      // successful runs
            for(int i=0; i<MAX_ITERS && sruns < ITERS; ++i)
            {
                
                // FullyDistVec ( shared_ptr<CommGrid> grid, IT globallen, NT initval);
                FullyDistVec<int64_t, ParentType> parents ( A.getcommgrid(), A.getncol(), ParentType());    
 
                // FullyDistSpVec ( shared_ptr<CommGrid> grid, IT glen);
                FullyDistSpVec<int64_t, ParentType> fringe(A.getcommgrid(), A.getncol());   
 
                MPI_Barrier(MPI_COMM_WORLD);
                double t1 = MPI_Wtime();
                
            #ifdef USE_PAPI
                char errorstring[PAPI_MAX_STR_LEN+1];
                long long ptr2values[combblas_papi_num_events];
            #endif
 
                fringe.SetElement(Cands[i], Cands[i]);
                parents.SetElement(Cands[i], ParentType(Cands[i])); // make root discovered
                int iterations = 0;
                while(fringe.getnnz() > 0)
                {
                #ifdef USE_PAPI
                    vector< vector<long long> > papi_this_iterate(num_bfs_papi_labels);
                #endif
                    fringe.ApplyInd(NumSetter);
                        
                #ifdef USE_PAPI
                    int errorcode = PAPI_start_counters(combblas_papi_events, combblas_papi_num_events);
                    CheckPAPI(errorcode, errorstring);
                    long_long papi_t_sta = PAPI_get_real_usec();
                #endif
                    
                    // SpMV with sparse vector, optimizations disabled for generality
                    SpMV<LatestRetwitterBFS>(A, fringe, fringe, false);
                    
                #ifdef USE_PAPI
                    lond_long papi_t_end = PAPI_get_real_usec();
                    errorcode = PAPI_read_counters(ptr2values, combblas_papi_num_events);
                    CheckPAPI(errorcode, errorstring);
 
                    errorcode = PAPI_stop_counters(ptr2values, combblas_papi_num_events);
                    papi_this_iterate[bfs_papi_enum.SpMV].resize(combblas_papi_num_events+1);   // +1 for the time
                    for(int k=0; k<combblas_papi_num_events; ++k)
                    {
                        papi_this_iterate[bfs_papi_enum.SpMV][k] = ptr2values[k];
                    }
                    papi_this_iterate[bfs_papi_enum.SpMV][combblas_papi_num_events] = papi_t_end - papi_s_end;  // time in usec
                        
                    PAPI_start_counters(combblas_papi_events, combblas_papi_num_events);
                    CheckPAPI(errorcode, errorstring);
                    papi_t_sta = PAPI_get_real_usec();
                #endif
                    
                    //  EWiseApply (const FullyDistSpVec<IU,NU1> & V, const FullyDistVec<IU,NU2> & W, 
                    //      _BinaryOperation _binary_op, _BinaryPredicate _doOp, bool allowVNulls, NU1 Vzero)
                    fringe = EWiseApply<ParentType>(fringe, parents, getfringe(), keepinfrontier_f(), true, ParentType());
                    
                #ifdef USE_PAPI
                    papi_t_end = PAPI_get_real_usec();
                    errorcode = PAPI_read_counters(ptr2values, combblas_papi_num_events);
                    CheckPAPI(errorcode, errorstring);
                    
                    errorcode = PAPI_stop_counters(ptr2values, combblas_papi_num_events);
                    papi_this_iterate[bfs_papi_enum.fringe_updt].resize(combblas_papi_num_events+1);    // +1 for the time
                    for(int k=0; k<combblas_papi_num_events; ++k)
                    {
                        papi_this_iterate[bfs_papi_enum.fringe_updt][k] = ptr2values[k];
                    }
                    papi_this_iterate[bfs_papi_enum.fringe_updt][combblas_papi_num_events] = papi_t_end - papi_s_end;   // time in usec
                    
                    PAPI_start_counters(combblas_papi_events, combblas_papi_num_events);
                    CheckPAPI(errorcode, errorstring);
                    papi_t_sta = PAPI_get_real_usec();      
                #endif
                    
                    parents += fringe;  // ABAB: Convert this to EwiseApply for compliance in PAPI timings
                
                #ifdef USE_PAPI
                    papi_t_end = PAPI_get_real_usec();
                    errorcode = PAPI_read_counters(ptr2values, combblas_papi_num_events);
                    CheckPAPI(errorcode, errorstring);      
                    
                    errorcode = PAPI_stop_counters(ptr2values, combblas_papi_num_events);
                    papi_this_iterate[bfs_papi_enum.parents_updt].resize(combblas_papi_num_events+1);   // +1 for the time
                    for(int k=0; k<combblas_papi_num_events; ++k)
                    {
                        papi_this_iterate[bfs_papi_enum.parents_updt][k] = ptr2values[k];
                    }
                    papi_this_iterate[bfs_papi_enum.parents_updt][combblas_papi_num_events] = papi_t_end - papi_s_end;  // time in usec
                    bfs_counters.push_back(papi_this_iterate);  // copy to bfs_counters[iterations]
                #endif
                    
                    iterations++;
                }
                MPI_Barrier(MPI_COMM_WORLD);
                double t2 = MPI_Wtime();
    
            
                FullyDistSpVec<int64_t, ParentType> parentsp = parents.Find(isparentset());
                parentsp.Apply(myset<ParentType>(ParentType(1)));
 
#ifndef ONLYTIME
                        FullyDistSpVec<int64_t, int64_t> intraversed(fullWorld);
                        FullyDistSpVec<int64_t, int64_t> inprocessed(fullWorld);
                        FullyDistSpVec<int64_t, int64_t> outraversed(fullWorld);
                        FullyDistSpVec<int64_t, int64_t> ouprocessed(fullWorld);
                inprocessed = EWiseApply<int64_t>(parentsp, indegrees, seldegree(), passifthere(), true, ParentType());
                ouprocessed = EWiseApply<int64_t>(parentsp, oudegrees, seldegree(), passifthere(), true, ParentType());
                int64_t nedges, in_nedges, ou_nedges;
                if(string(argv[1]) == string("Gen"))
                {
                    FullyDistSpVec<int64_t, int64_t> traversed = EWiseApply<int64_t>(parentsp, degrees_filt[trials], seldegree(), passifthere(), true, ParentType());
                    nedges = traversed.Reduce(plus<int64_t>(), (int64_t) 0);
                }
                else 
                {
                    intraversed = EWiseApply<int64_t>(parentsp, indegrees_filt, seldegree(), passifthere(), true, ParentType());
                    outraversed = EWiseApply<int64_t>(parentsp, oudegrees_filt, seldegree(), passifthere(), true, ParentType());
                    in_nedges = intraversed.Reduce(plus<int64_t>(), (int64_t) 0);
                    ou_nedges = outraversed.Reduce(plus<int64_t>(), (int64_t) 0);
                    nedges = in_nedges + ou_nedges; // count birectional edges twice
                }
                
                int64_t in_nedges_processed = inprocessed.Reduce(plus<int64_t>(), (int64_t) 0);
                int64_t ou_nedges_processed = ouprocessed.Reduce(plus<int64_t>(), (int64_t) 0);
                int64_t nedges_processed = in_nedges_processed + ou_nedges_processed;   // count birectional edges twice
#else
                int64_t in_nedges, ou_nedges, nedges, in_nedges_processed, ou_nedges_processed, nedges_processed = 0;
#endif
 
                if(parentsp.getnnz() > CC_LIMIT)
                {
                    // intraversed.PrintInfo("Incoming edges traversed per vertex");
                    // intraversed.DebugPrint();
                    // outraversed.PrintInfo("Outgoing edges traversed per vertex");
                    // outraversed.DebugPrint();
                    
                #ifdef DEBUG
                    parents.PrintInfo("Final parents array");
                    parents.DebugPrint();
                #endif
                    
                    ostringstream outnew;  
                    outnew << i << "th starting vertex was " << Cands[i] << endl;
                    outnew << "Number iterations: " << iterations << endl;
                    outnew << "Number of vertices found: " << parentsp.getnnz() << endl; 
                    outnew << "Number of edges traversed in both directions: " << nedges << endl;
                    if(string(argv[1]) == string("File"))
                        outnew << "Number of edges traversed in one direction: " << ou_nedges << endl;
                    outnew << "Number of edges processed in both directions: " << nedges_processed << endl;
                    outnew << "Number of edges processed in one direction: " << ou_nedges_processed << endl;
                    outnew << "BFS time: " << t2-t1 << " seconds" << endl;
                    outnew << "MTEPS (bidirectional): " << static_cast<double>(nedges) / (t2-t1) / 1000000.0 << endl;
                    if(string(argv[1]) == string("File"))
                        outnew << "MTEPS (unidirectional): " << static_cast<double>(ou_nedges) / (t2-t1) / 1000000.0 << endl;
                    outnew << "MPEPS (bidirectional): " << static_cast<double>(nedges_processed) / (t2-t1) / 1000000.0 << endl;
                    outnew << "MPEPS (unidirectional): " << static_cast<double>(ou_nedges_processed) / (t2-t1) / 1000000.0 << endl;
                    outnew << "Total communication (average so far): " << (cblas_allgathertime + cblas_alltoalltime) / (i+1) << endl;
                    
                    /* Write to PAPI */
                #ifdef USE_PAPI
                    ostringstream papiout;
                    papiout << i << "th starting vertex was " << Cands[i] << endl;
                    papiout << "Threshold is " << LatestRetwitterBFS::sincedate  << endl;
                    
                    for(int i=0; i < iterations; i++)   // over all spmv iterations in this BFS  
                    {
                        papiout << "Iteration : " << i << endl;
                        for(int j=0; j < bfs_papi_labels; ++j)
                        {
                            papiout << "Function : " << bfs_papi_labels[j] << endl;
 
                            for(int k=0; k < combblas_papi_num_events; ++k)
                            {
                                papiout << combblas_event_names[k] << ":\t" << bfs_counters[i][j][k] << endl; 
                            }
                            papiout << "Time (usec)" << ":\t" << bfs_counters[i][j][combblas_papi_num_events] << endl; 
                        }
                    }
                    SpParHelper::PrintFile(papiout.str(), "PAPIRES.txt");
                #endif
                    /* (End) Write to PAPI */   
 
                    TIMES[sruns] = t2-t1;
                    if(string(argv[1]) == string("Gen"))
                        EDGES[sruns] = static_cast<double>(nedges);
                    else
                        EDGES[sruns] = static_cast<double>(ou_nedges);
 
                    MTEPS[sruns] = EDGES[sruns] / (t2-t1) / 1000000.0;
                    MPEPS[sruns++] = static_cast<double>(nedges_processed) / (t2-t1) / 1000000.0;
                    SpParHelper::Print(outnew.str());
                }
            }
            if (sruns < MINRUNS)
            {
                SpParHelper::Print("Not enough valid runs done\n");
                MPI_Finalize();
                return 0;
            }
            ostringstream os;
            
            os << sruns << " valid runs done" << endl;
            os << "Connected component lower limite was " << CC_LIMIT << endl;
            os << "Per iteration communication times: " << endl;
            os << "AllGatherv: " << cblas_allgathertime / sruns << endl;
            os << "AlltoAllv: " << cblas_alltoalltime / sruns << endl;
 
            sort(EDGES, EDGES+sruns);
            os << "--------------------------" << endl;
            os << "Min nedges: " << EDGES[0] << endl;
            os << "Median nedges: " << (EDGES[(sruns/2)-1] + EDGES[sruns/2])/2 << endl;
            os << "Max nedges: " << EDGES[sruns-1] << endl;
            double mean = accumulate( EDGES, EDGES+sruns, 0.0 )/ sruns;
            vector<double> zero_mean(sruns);    // find distances to the mean
            transform(EDGES, EDGES+sruns, zero_mean.begin(), bind2nd( minus<double>(), mean ));     
            // self inner-product is sum of sum of squares
            double deviation = inner_product( zero_mean.begin(),zero_mean.end(), zero_mean.begin(), 0.0 );
            deviation = sqrt( deviation / (sruns-1) );
            os << "Mean nedges: " << mean << endl;
            os << "STDDEV nedges: " << deviation << endl;
            os << "--------------------------" << endl;
    
            sort(TIMES,TIMES+sruns);
            os << "Filter keeps " << static_cast<double>(keep[trials])/100.0 << " percentage of edges" << endl;
            os << "Min time: " << TIMES[0] << " seconds" << endl;
            os << "Median time: " << (TIMES[(sruns/2)-1] + TIMES[sruns/2])/2 << " seconds" << endl;
            os << "Max time: " << TIMES[sruns-1] << " seconds" << endl;
            mean = accumulate( TIMES, TIMES+sruns, 0.0 )/ sruns;
            transform(TIMES, TIMES+sruns, zero_mean.begin(), bind2nd( minus<double>(), mean ));     
            deviation = inner_product( zero_mean.begin(),zero_mean.end(), zero_mean.begin(), 0.0 );
            deviation = sqrt( deviation / (sruns-1) );
            os << "Mean time: " << mean << " seconds" << endl;
            os << "STDDEV time: " << deviation << " seconds" << endl;
            os << "--------------------------" << endl;
 
            sort(MTEPS, MTEPS+sruns);
            os << "Min MTEPS: " << MTEPS[0] << endl;
            os << "Median MTEPS: " << (MTEPS[(sruns/2)-1] + MTEPS[sruns/2])/2 << endl;
            os << "Max MTEPS: " << MTEPS[sruns-1] << endl;
            transform(MTEPS, MTEPS+sruns, INVMTEPS, safemultinv<double>());     // returns inf for zero teps
            double hteps = static_cast<double>(sruns) / accumulate(INVMTEPS, INVMTEPS+sruns, 0.0);  
            os << "Harmonic mean of MTEPS: " << hteps << endl;
            transform(INVMTEPS, INVMTEPS+sruns, zero_mean.begin(), bind2nd(minus<double>(), 1/hteps));
            deviation = inner_product( zero_mean.begin(),zero_mean.end(), zero_mean.begin(), 0.0 );
            deviation = sqrt( deviation / (sruns-1) ) * (hteps*hteps);  // harmonic_std_dev
            os << "Harmonic standard deviation of MTEPS: " << deviation << endl;
 
            sort(MPEPS, MPEPS+sruns);
            os << "Bidirectional Processed Edges per second (to estimate sustained BW)"<< endl;
            os << "Min MPEPS: " << MPEPS[0] << endl;
            os << "Median MPEPS: " << (MPEPS[(sruns/2)-1] + MPEPS[sruns/2])/2 << endl;
            os << "Max MPEPS: " << MPEPS[sruns-1] << endl;
            transform(MPEPS, MPEPS+sruns, INVMPEPS, safemultinv<double>());     // returns inf for zero teps
            double hpeps = static_cast<double>(sruns) / accumulate(INVMPEPS, INVMPEPS+sruns, 0.0);  
            os << "Harmonic mean of MPEPS: " << hpeps << endl;
            transform(INVMPEPS, INVMPEPS+sruns, zero_mean.begin(), bind2nd(minus<double>(), 1/hpeps));
            deviation = inner_product( zero_mean.begin(),zero_mean.end(), zero_mean.begin(), 0.0 );
            deviation = sqrt( deviation / (sruns-1) ) * (hpeps*hpeps);  // harmonic_std_dev
            os << "Harmonic standard deviation of MPEPS: " << deviation << endl;
            SpParHelper::Print(os.str());
        }
    }
    MPI_Finalize();
    return 0;
}