\contentsline {chapter}{\numberline {1}Introduction to Parallel Processing}{1}{chapter.1}
\contentsline {section}{\numberline {1.1}Platforms}{1}{section.1.1}
\contentsline {subsection}{\numberline {1.1.1}Why R?}{1}{subsection.1.1.1}
\contentsline {section}{\numberline {1.2}Why Use Parallel Systems?}{1}{section.1.2}
\contentsline {subsection}{\numberline {1.2.1}Execution Speed}{1}{subsection.1.2.1}
\contentsline {subsection}{\numberline {1.2.2}Memory}{2}{subsection.1.2.2}
\contentsline {subsection}{\numberline {1.2.3}Distributed Processing}{3}{subsection.1.2.3}
\contentsline {subsection}{\numberline {1.2.4}Our Focus Here}{3}{subsection.1.2.4}
\contentsline {section}{\numberline {1.3}Parallel Processing Hardware}{3}{section.1.3}
\contentsline {subsection}{\numberline {1.3.1}Shared-Memory Systems}{3}{subsection.1.3.1}
\contentsline {subsubsection}{\numberline {1.3.1.1}Basic Architecture}{3}{subsubsection.1.3.1.1}
\contentsline {subsubsection}{\numberline {1.3.1.2}Multiprocessor Topologies}{4}{subsubsection.1.3.1.2}
\contentsline {subsubsection}{\numberline {1.3.1.3}Memory Issues Etc.}{4}{subsubsection.1.3.1.3}
\contentsline {subsection}{\numberline {1.3.2}Message-Passing Systems}{5}{subsection.1.3.2}
\contentsline {subsubsection}{\numberline {1.3.2.1}Basic Architecture}{5}{subsubsection.1.3.2.1}
\contentsline {subsubsection}{\numberline {1.3.2.2}Example: Clusters}{5}{subsubsection.1.3.2.2}
\contentsline {subsection}{\numberline {1.3.3}SIMD}{5}{subsection.1.3.3}
\contentsline {section}{\numberline {1.4}Programmer World Views}{6}{section.1.4}
\contentsline {subsection}{\numberline {1.4.1}Example: Matrix-Vector Multiply}{6}{subsection.1.4.1}
\contentsline {subsection}{\numberline {1.4.2}Shared-Memory}{6}{subsection.1.4.2}
\contentsline {subsubsection}{\numberline {1.4.2.1}Programmer View}{6}{subsubsection.1.4.2.1}
\contentsline {section}{\numberline {1.5}Shared Memory Programming}{7}{section.1.5}
\contentsline {subsection}{\numberline {1.5.1}Low-Level Threads Systems: Pthreads}{7}{subsection.1.5.1}
\contentsline {subsubsection}{\numberline {1.5.1.1}Pthreads Example: Finding Primes}{8}{subsubsection.1.5.1.1}
\contentsline {subsection}{\numberline {1.5.2}Compiling and Linking}{10}{subsection.1.5.2}
\contentsline {subsection}{\numberline {1.5.3}The Notion of a ``Critical Section''}{10}{subsection.1.5.3}
\contentsline {subsection}{\numberline {1.5.4}Role of the OS}{13}{subsection.1.5.4}
\contentsline {subsection}{\numberline {1.5.5}Debugging Threads Programs}{13}{subsection.1.5.5}
\contentsline {section}{\numberline {1.6}A Note on Global Variables}{14}{section.1.6}
\contentsline {subsection}{\numberline {1.6.1}Higher-Level Threads Programming: OpenMP}{15}{subsection.1.6.1}
\contentsline {subsubsection}{\numberline {1.6.1.1}Example: Sampling Bucket Sort}{15}{subsubsection.1.6.1.1}
\contentsline {subsubsection}{\numberline {1.6.1.2}Debugging OpenMP}{18}{subsubsection.1.6.1.2}
\contentsline {subsection}{\numberline {1.6.2}Message Passing}{19}{subsection.1.6.2}
\contentsline {subsubsection}{\numberline {1.6.2.1}Programmer View}{19}{subsubsection.1.6.2.1}
\contentsline {subsubsection}{\numberline {1.6.2.2}Example: MPI Prime Numbers Finder}{19}{subsubsection.1.6.2.2}
\contentsline {subsection}{\numberline {1.6.3}Scatter/Gather}{22}{subsection.1.6.3}
\contentsline {subsubsection}{\numberline {1.6.3.1}R snow Package}{23}{subsubsection.1.6.3.1}
\contentsline {section}{\numberline {1.7}Threads Programming in R: Rdsm}{27}{section.1.7}
\contentsline {subsection}{\numberline {1.7.1}Example: Matrix Multiplication}{27}{subsection.1.7.1}
\contentsline {subsection}{\numberline {1.7.2}Example: Maximal Burst in a Time Series}{28}{subsection.1.7.2}
\contentsline {chapter}{\numberline {2}Recurring Performance Issues}{31}{chapter.2}
\contentsline {section}{\numberline {2.1}Communication Bottlenecks}{31}{section.2.1}
\contentsline {section}{\numberline {2.2}Load Balancing}{32}{section.2.2}
\contentsline {section}{\numberline {2.3}``Embarrassingly Parallel'' Applications}{32}{section.2.3}
\contentsline {subsection}{\numberline {2.3.1}What People Mean by ``Embarrassingly Parallel''}{32}{subsection.2.3.1}
\contentsline {subsection}{\numberline {2.3.2}Iterative Algorithms}{33}{subsection.2.3.2}
\contentsline {section}{\numberline {2.4}Static (But Possibly Random) Task Assignment Typically Better Than Dynamic}{34}{section.2.4}
\contentsline {subsection}{\numberline {2.4.1}Example: Matrix-Vector Multiply}{34}{subsection.2.4.1}
\contentsline {subsection}{\numberline {2.4.2}Load Balance, Revisited}{36}{subsection.2.4.2}
\contentsline {subsection}{\numberline {2.4.3}Example: Mutual Web Outlinks}{37}{subsection.2.4.3}
\contentsline {subsection}{\numberline {2.4.4}Work Stealing}{38}{subsection.2.4.4}
\contentsline {subsection}{\numberline {2.4.5}Timing Example}{38}{subsection.2.4.5}
\contentsline {subsection}{\numberline {2.4.6}Example: Extracting a Submatrix}{38}{subsection.2.4.6}
\contentsline {section}{\numberline {2.5}Latency and Bandwidth}{39}{section.2.5}
\contentsline {section}{\numberline {2.6}Relative Merits: Performance of Shared-Memory Vs. Message-Passing}{40}{section.2.6}
\contentsline {section}{\numberline {2.7}Memory Allocation Issues}{41}{section.2.7}
\contentsline {section}{\numberline {2.8}Issues Particular to Shared-Memory Systems}{41}{section.2.8}
\contentsline {chapter}{\numberline {3}Shared Memory Parallelism}{43}{chapter.3}
\contentsline {section}{\numberline {3.1}What Is Shared?}{43}{section.3.1}
\contentsline {section}{\numberline {3.2}Memory Modules}{44}{section.3.2}
\contentsline {subsection}{\numberline {3.2.1}Interleaving}{44}{subsection.3.2.1}
\contentsline {subsection}{\numberline {3.2.2}Bank Conflicts and Solutions}{45}{subsection.3.2.2}
\contentsline {subsection}{\numberline {3.2.3}Example: Code to Implement Padding}{47}{subsection.3.2.3}
\contentsline {section}{\numberline {3.3}Interconnection Topologies}{48}{section.3.3}
\contentsline {subsection}{\numberline {3.3.1}SMP Systems}{48}{subsection.3.3.1}
\contentsline {subsection}{\numberline {3.3.2}NUMA Systems}{49}{subsection.3.3.2}
\contentsline {subsection}{\numberline {3.3.3}NUMA Interconnect Topologies}{50}{subsection.3.3.3}
\contentsline {subsubsection}{\numberline {3.3.3.1}Crossbar Interconnects}{50}{subsubsection.3.3.3.1}
\contentsline {subsubsection}{\numberline {3.3.3.2}Omega (or Delta) Interconnects}{52}{subsubsection.3.3.3.2}
\contentsline {subsection}{\numberline {3.3.4}Comparative Analysis}{53}{subsection.3.3.4}
\contentsline {subsection}{\numberline {3.3.5}Why Have Memory in Modules?}{54}{subsection.3.3.5}
\contentsline {section}{\numberline {3.4}Synchronization Hardware}{55}{section.3.4}
\contentsline {subsection}{\numberline {3.4.1}Test-and-Set Instructions}{55}{subsection.3.4.1}
\contentsline {subsubsection}{\numberline {3.4.1.1}LOCK Prefix on Intel Processors}{55}{subsubsection.3.4.1.1}
\contentsline {subsubsection}{\numberline {3.4.1.2}Locks with More Complex Interconnects}{57}{subsubsection.3.4.1.2}
\contentsline {subsection}{\numberline {3.4.2}May Not Need the Latest}{57}{subsection.3.4.2}
\contentsline {subsection}{\numberline {3.4.3}Fetch-and-Add Instructions}{57}{subsection.3.4.3}
\contentsline {section}{\numberline {3.5}Cache Issues}{58}{section.3.5}
\contentsline {subsection}{\numberline {3.5.1}Cache Coherency}{58}{subsection.3.5.1}
\contentsline {subsection}{\numberline {3.5.2}Example: the MESI Cache Coherency Protocol}{61}{subsection.3.5.2}
\contentsline {subsection}{\numberline {3.5.3}The Problem of ``False Sharing''}{63}{subsection.3.5.3}
\contentsline {section}{\numberline {3.6}Memory-Access Consistency Policies}{64}{section.3.6}
\contentsline {section}{\numberline {3.7}Fetch-and-Add Combining within Interconnects}{66}{section.3.7}
\contentsline {section}{\numberline {3.8}Multicore Chips}{66}{section.3.8}
\contentsline {section}{\numberline {3.9}Optimal Number of Threads}{67}{section.3.9}
\contentsline {section}{\numberline {3.10}Processor Affinity}{67}{section.3.10}
\contentsline {section}{\numberline {3.11}Illusion of Shared-Memory through Software}{68}{section.3.11}
\contentsline {subsection}{\numberline {3.11.1}Software Distributed Shared Memory}{68}{subsection.3.11.1}
\contentsline {subsection}{\numberline {3.11.2}Case Study: JIAJIA}{70}{subsection.3.11.2}
\contentsline {section}{\numberline {3.12}Barrier Implementation}{73}{section.3.12}
\contentsline {subsection}{\numberline {3.12.1}A Use-Once Version}{74}{subsection.3.12.1}
\contentsline {subsection}{\numberline {3.12.2}An Attempt to Write a Reusable Version}{75}{subsection.3.12.2}
\contentsline {subsection}{\numberline {3.12.3}A Correct Version}{75}{subsection.3.12.3}
\contentsline {subsection}{\numberline {3.12.4}Refinements}{76}{subsection.3.12.4}
\contentsline {subsubsection}{\numberline {3.12.4.1}Use of Wait Operations}{76}{subsubsection.3.12.4.1}
\contentsline {subsubsection}{\numberline {3.12.4.2}Parallelizing the Barrier Operation}{78}{subsubsection.3.12.4.2}
\contentsline {paragraph}{\numberline {3.12.4.2.1}Tree Barriers}{78}{paragraph.3.12.4.2.1}
\contentsline {paragraph}{\numberline {3.12.4.2.2}Butterfly Barriers}{78}{paragraph.3.12.4.2.2}
\contentsline {chapter}{\numberline {4}Introduction to OpenMP}{81}{chapter.4}
\contentsline {section}{\numberline {4.1}Overview}{81}{section.4.1}
\contentsline {section}{\numberline {4.2}Example: Dijkstra Shortest-Path Algorithm}{81}{section.4.2}
\contentsline {subsection}{\numberline {4.2.1}The Algorithm}{84}{subsection.4.2.1}
\contentsline {subsection}{\numberline {4.2.2}The OpenMP {\tt parallel} Pragma}{84}{subsection.4.2.2}
\contentsline {subsection}{\numberline {4.2.3}Scope Issues}{85}{subsection.4.2.3}
\contentsline {subsection}{\numberline {4.2.4}The OpenMP {\tt single} Pragma}{86}{subsection.4.2.4}
\contentsline {subsection}{\numberline {4.2.5}The OpenMP {\tt barrier} Pragma}{86}{subsection.4.2.5}
\contentsline {subsection}{\numberline {4.2.6}Implicit Barriers}{87}{subsection.4.2.6}
\contentsline {subsection}{\numberline {4.2.7}The OpenMP {\tt critical} Pragma}{87}{subsection.4.2.7}
\contentsline {section}{\numberline {4.3}The OpenMP {\tt for} Pragma}{87}{section.4.3}
\contentsline {subsection}{\numberline {4.3.1}Example: Dijkstra with Parallel for Loops}{87}{subsection.4.3.1}
\contentsline {subsection}{\numberline {4.3.2}Nested Loops}{90}{subsection.4.3.2}
\contentsline {subsection}{\numberline {4.3.3}Controlling the Partitioning of Work to Threads: the schedule Clause}{90}{subsection.4.3.3}
\contentsline {subsection}{\numberline {4.3.4}Example: In-Place Matrix Transpose}{92}{subsection.4.3.4}
\contentsline {subsection}{\numberline {4.3.5}The OpenMP {\tt reduction} Clause}{93}{subsection.4.3.5}
\contentsline {section}{\numberline {4.4}Example: Mandelbrot Set}{94}{section.4.4}
\contentsline {section}{\numberline {4.5}The Task Directive}{97}{section.4.5}
\contentsline {subsection}{\numberline {4.5.1}Example: Quicksort}{98}{subsection.4.5.1}
\contentsline {section}{\numberline {4.6}Other OpenMP Synchronization Issues}{99}{section.4.6}
\contentsline {subsection}{\numberline {4.6.1}The OpenMP {\tt atomic} Clause}{99}{subsection.4.6.1}
\contentsline {subsection}{\numberline {4.6.2}Memory Consistency and the {\tt flush} Pragma}{100}{subsection.4.6.2}
\contentsline {section}{\numberline {4.7}Combining Work-Sharing Constructs}{101}{section.4.7}
\contentsline {section}{\numberline {4.8}The Rest of OpenMP}{102}{section.4.8}
\contentsline {section}{\numberline {4.9}Compiling, Running and Debugging OpenMP Code}{102}{section.4.9}
\contentsline {subsection}{\numberline {4.9.1}Compiling}{102}{subsection.4.9.1}
\contentsline {subsection}{\numberline {4.9.2}Running}{103}{subsection.4.9.2}
\contentsline {subsection}{\numberline {4.9.3}Debugging}{103}{subsection.4.9.3}
\contentsline {section}{\numberline {4.10}Performance}{104}{section.4.10}
\contentsline {subsection}{\numberline {4.10.1}The Effect of Problem Size}{104}{subsection.4.10.1}
\contentsline {subsection}{\numberline {4.10.2}Some Fine Tuning}{104}{subsection.4.10.2}
\contentsline {subsection}{\numberline {4.10.3}OpenMP Internals}{108}{subsection.4.10.3}
\contentsline {section}{\numberline {4.11}Example: Root Finding}{109}{section.4.11}
\contentsline {section}{\numberline {4.12}Example: Mutual Outlinks}{110}{section.4.12}
\contentsline {section}{\numberline {4.13}Example: Transforming an Adjacency Matrix}{112}{section.4.13}
\contentsline {section}{\numberline {4.14}Example: Finding the Maximal Burst in a Time Series}{115}{section.4.14}
\contentsline {section}{\numberline {4.15}Locks with OpenMP}{117}{section.4.15}
\contentsline {section}{\numberline {4.16}Other Examples of OpenMP Code in This Book}{117}{section.4.16}
\contentsline {chapter}{\numberline {5}Introduction to GPU Programming with CUDA}{119}{chapter.5}
\contentsline {section}{\numberline {5.1}Overview}{119}{section.5.1}
\contentsline {section}{\numberline {5.2}Some Terminology}{120}{section.5.2}
\contentsline {section}{\numberline {5.3}Example: Calculate Row Sums}{120}{section.5.3}
\contentsline {section}{\numberline {5.4}Understanding the Hardware Structure}{124}{section.5.4}
\contentsline {subsection}{\numberline {5.4.1}Processing Units}{124}{subsection.5.4.1}
\contentsline {subsection}{\numberline {5.4.2}Thread Operation}{125}{subsection.5.4.2}
\contentsline {subsubsection}{\numberline {5.4.2.1}SIMT Architecture}{125}{subsubsection.5.4.2.1}
\contentsline {subsubsection}{\numberline {5.4.2.2}The Problem of Thread Divergence}{125}{subsubsection.5.4.2.2}
\contentsline {subsubsection}{\numberline {5.4.2.3}``OS in Hardware''}{125}{subsubsection.5.4.2.3}
\contentsline {subsection}{\numberline {5.4.3}Memory Structure}{126}{subsection.5.4.3}
\contentsline {subsubsection}{\numberline {5.4.3.1}Shared and Global Memory}{126}{subsubsection.5.4.3.1}
\contentsline {subsubsection}{\numberline {5.4.3.2}Global-Memory Performance Issues}{130}{subsubsection.5.4.3.2}
\contentsline {subsubsection}{\numberline {5.4.3.3}Shared-Memory Performance Issues}{130}{subsubsection.5.4.3.3}
\contentsline {subsubsection}{\numberline {5.4.3.4}Host/Device Memory Transfer Performance Issues}{131}{subsubsection.5.4.3.4}
\contentsline {subsubsection}{\numberline {5.4.3.5}Other Types of Memory}{131}{subsubsection.5.4.3.5}
\contentsline {subsection}{\numberline {5.4.4}Threads Hierarchy}{133}{subsection.5.4.4}
\contentsline {subsection}{\numberline {5.4.5}What's NOT There}{135}{subsection.5.4.5}
\contentsline {section}{\numberline {5.5}Synchronization, Within and Between Blocks}{135}{section.5.5}
\contentsline {section}{\numberline {5.6}More on the Blocks/Threads Tradeoff}{136}{section.5.6}
\contentsline {section}{\numberline {5.7}Hardware Requirements, Installation, Compilation, Debugging}{137}{section.5.7}
\contentsline {section}{\numberline {5.8}Example: Improving the Row Sums Program}{139}{section.5.8}
\contentsline {section}{\numberline {5.9}Example: Finding the Mean Number of Mutual Outlinks}{141}{section.5.9}
\contentsline {section}{\numberline {5.10}Example: Finding Prime Numbers}{142}{section.5.10}
\contentsline {section}{\numberline {5.11}Example: Finding Cumulative Sums}{145}{section.5.11}
\contentsline {section}{\numberline {5.12}When Is It Advantageous to Use Shared Memory}{147}{section.5.12}
\contentsline {section}{\numberline {5.13}Example: Transforming an Adjacency Matrix}{147}{section.5.13}
\contentsline {section}{\numberline {5.14}Error Checking}{150}{section.5.14}
\contentsline {section}{\numberline {5.15}Loop Unrolling}{151}{section.5.15}
\contentsline {section}{\numberline {5.16}Short Vectors}{152}{section.5.16}
\contentsline {section}{\numberline {5.17}Newer Generations}{152}{section.5.17}
\contentsline {subsection}{\numberline {5.17.1}True Caching}{152}{subsection.5.17.1}
\contentsline {subsection}{\numberline {5.17.2}Unified Memory}{152}{subsection.5.17.2}
\contentsline {section}{\numberline {5.18}CUDA from a Higher Level}{153}{section.5.18}
\contentsline {subsection}{\numberline {5.18.1}CUBLAS}{153}{subsection.5.18.1}
\contentsline {subsubsection}{\numberline {5.18.1.1}Example: Row Sums Once Again}{153}{subsubsection.5.18.1.1}
\contentsline {subsection}{\numberline {5.18.2}Thrust}{155}{subsection.5.18.2}
\contentsline {subsection}{\numberline {5.18.3}CUFFT}{156}{subsection.5.18.3}
\contentsline {section}{\numberline {5.19}Other CUDA Examples in This Book}{156}{section.5.19}
\contentsline {chapter}{\numberline {6}Introduction to Thrust Programming}{157}{chapter.6}
\contentsline {section}{\numberline {6.1}Compiling Thrust Code}{157}{section.6.1}
\contentsline {subsection}{\numberline {6.1.1}Compiling to CUDA}{157}{subsection.6.1.1}
\contentsline {subsection}{\numberline {6.1.2}Compiling to OpenMP}{158}{subsection.6.1.2}
\contentsline {section}{\numberline {6.2}Example: Counting the Number of Unique Values in an Array}{158}{section.6.2}
\contentsline {section}{\numberline {6.3}Example: A Plain-C Wrapper for Thrust sort()}{162}{section.6.3}
\contentsline {section}{\numberline {6.4}Example: Calculating Percentiles in an Array}{163}{section.6.4}
\contentsline {section}{\numberline {6.5}Mixing Thrust and CUDA Code}{165}{section.6.5}
\contentsline {section}{\numberline {6.6}Example: Doubling Every k$^{th}$ Element of an Array}{166}{section.6.6}
\contentsline {section}{\numberline {6.7}Scatter and Gather Operations}{167}{section.6.7}
\contentsline {subsection}{\numberline {6.7.1}Example: Matrix Transpose}{169}{subsection.6.7.1}
\contentsline {section}{\numberline {6.8}Advanced (``Fancy'') Iterators}{170}{section.6.8}
\contentsline {subsection}{\numberline {6.8.1}Example: Matrix Transpose Again}{170}{subsection.6.8.1}
\contentsline {section}{\numberline {6.9}A Timing Comparison}{172}{section.6.9}
\contentsline {section}{\numberline {6.10}Example: Transforming an Adjacency Matrix}{176}{section.6.10}
\contentsline {section}{\numberline {6.11}Prefix Scan}{178}{section.6.11}
\contentsline {section}{\numberline {6.12}More on Use of Thrust for a CUDA Backend}{179}{section.6.12}
\contentsline {subsection}{\numberline {6.12.1}Synchronicity}{179}{subsection.6.12.1}
\contentsline {section}{\numberline {6.13}Error Messages}{179}{section.6.13}
\contentsline {section}{\numberline {6.14}Other Examples of Thrust Code in This Book}{180}{section.6.14}
\contentsline {chapter}{\numberline {7}Message Passing Systems}{181}{chapter.7}
\contentsline {section}{\numberline {7.1}Overview}{181}{section.7.1}
\contentsline {section}{\numberline {7.2}A Historical Example: Hypercubes}{182}{section.7.2}
\contentsline {subsection}{\numberline {7.2.1}Definitions}{182}{subsection.7.2.1}
\contentsline {section}{\numberline {7.3}Networks of Workstations (NOWs)}{184}{section.7.3}
\contentsline {subsection}{\numberline {7.3.1}The Network Is Literally the Weakest Link}{184}{subsection.7.3.1}
\contentsline {subsection}{\numberline {7.3.2}Other Issues}{185}{subsection.7.3.2}
\contentsline {section}{\numberline {7.4}Scatter/Gather Operations}{185}{section.7.4}
\contentsline {chapter}{\numberline {8}Introduction to MPI}{187}{chapter.8}
\contentsline {section}{\numberline {8.1}Overview}{187}{section.8.1}
\contentsline {subsection}{\numberline {8.1.1}History}{187}{subsection.8.1.1}
\contentsline {subsection}{\numberline {8.1.2}Structure and Execution}{188}{subsection.8.1.2}
\contentsline {subsection}{\numberline {8.1.3}Implementations}{188}{subsection.8.1.3}
\contentsline {subsection}{\numberline {8.1.4}Performance Issues}{188}{subsection.8.1.4}
\contentsline {section}{\numberline {8.2}Review of Earlier Example}{189}{section.8.2}
\contentsline {section}{\numberline {8.3}Example: Dijkstra Algorithm}{189}{section.8.3}
\contentsline {subsection}{\numberline {8.3.1}The Algorithm}{189}{subsection.8.3.1}
\contentsline {subsection}{\numberline {8.3.2}The MPI Code}{190}{subsection.8.3.2}
\contentsline {subsection}{\numberline {8.3.3}Introduction to MPI APIs}{193}{subsection.8.3.3}
\contentsline {subsubsection}{\numberline {8.3.3.1}MPI\_Init() and MPI\_Finalize()}{193}{subsubsection.8.3.3.1}
\contentsline {subsubsection}{\numberline {8.3.3.2}MPI\_Comm\_size() and MPI\_Comm\_rank()}{193}{subsubsection.8.3.3.2}
\contentsline {subsubsection}{\numberline {8.3.3.3}MPI\_Send()}{194}{subsubsection.8.3.3.3}
\contentsline {subsubsection}{\numberline {8.3.3.4}MPI\_Recv()}{195}{subsubsection.8.3.3.4}
\contentsline {section}{\numberline {8.4}Example: Removing 0s from an Array}{196}{section.8.4}
\contentsline {section}{\numberline {8.5}Debugging MPI Code}{197}{section.8.5}
\contentsline {section}{\numberline {8.6}Collective Communications}{198}{section.8.6}
\contentsline {subsection}{\numberline {8.6.1}Example: Refined Dijkstra Code}{198}{subsection.8.6.1}
\contentsline {subsection}{\numberline {8.6.2}MPI\_Bcast()}{201}{subsection.8.6.2}
\contentsline {subsection}{\numberline {8.6.3}MPI\_Reduce()/MPI\_Allreduce()}{202}{subsection.8.6.3}
\contentsline {subsection}{\numberline {8.6.4}MPI\_Gather()/MPI\_Allgather()}{203}{subsection.8.6.4}
\contentsline {subsection}{\numberline {8.6.5}The MPI\_Scatter()}{203}{subsection.8.6.5}
\contentsline {subsection}{\numberline {8.6.6}Example: Count the Number of Edges in a Directed Graph}{204}{subsection.8.6.6}
\contentsline {subsection}{\numberline {8.6.7}Example: Cumulative Sums}{204}{subsection.8.6.7}
\contentsline {subsection}{\numberline {8.6.8}Example: an MPI Solution to the Mutual Outlinks Problem}{206}{subsection.8.6.8}
\contentsline {subsection}{\numberline {8.6.9}The MPI\_Barrier()}{208}{subsection.8.6.9}
\contentsline {subsection}{\numberline {8.6.10}Creating Communicators}{208}{subsection.8.6.10}
\contentsline {section}{\numberline {8.7}Buffering, Synchrony and Related Issues}{209}{section.8.7}
\contentsline {subsection}{\numberline {8.7.1}Buffering, Etc.}{209}{subsection.8.7.1}
\contentsline {subsection}{\numberline {8.7.2}Safety}{210}{subsection.8.7.2}
\contentsline {subsection}{\numberline {8.7.3}Living Dangerously}{211}{subsection.8.7.3}
\contentsline {subsection}{\numberline {8.7.4}Safe Exchange Operations}{211}{subsection.8.7.4}
\contentsline {section}{\numberline {8.8}Use of MPI from Other Languages}{211}{section.8.8}
\contentsline {section}{\numberline {8.9}Other MPI Examples in This Book}{212}{section.8.9}
\contentsline {chapter}{\numberline {9}MapReduce Computation}{213}{chapter.9}
\contentsline {section}{\numberline {9.1}Apache Hadoop}{214}{section.9.1}
\contentsline {subsection}{\numberline {9.1.1}Hadoop Streaming}{214}{subsection.9.1.1}
\contentsline {subsection}{\numberline {9.1.2}Example: Word Count}{214}{subsection.9.1.2}
\contentsline {subsection}{\numberline {9.1.3}Running the Code}{215}{subsection.9.1.3}
\contentsline {subsection}{\numberline {9.1.4}Analysis of the Code}{216}{subsection.9.1.4}
\contentsline {subsection}{\numberline {9.1.5}Role of Disk Files}{217}{subsection.9.1.5}
\contentsline {section}{\numberline {9.2}Other MapReduce Systems}{218}{section.9.2}
\contentsline {section}{\numberline {9.3}R Interfaces to MapReduce Systems}{218}{section.9.3}
\contentsline {section}{\numberline {9.4}An Alternative: ``Snowdoop''}{218}{section.9.4}
\contentsline {subsection}{\numberline {9.4.1}Example: Snowdoop Word Count}{219}{subsection.9.4.1}
\contentsline {subsection}{\numberline {9.4.2}Example: Snowdoop k-Means Clustering}{220}{subsection.9.4.2}
\contentsline {chapter}{\numberline {10}The Parallel Prefix Problem}{223}{chapter.10}
\contentsline {section}{\numberline {10.1}Example: Permutations}{223}{section.10.1}
\contentsline {section}{\numberline {10.2}General Strategies for Parallel Scan Computation}{224}{section.10.2}
\contentsline {section}{\numberline {10.3}Implementations}{227}{section.10.3}
\contentsline {section}{\numberline {10.4}Example: Parallel Prefix Summing in OpenMP}{227}{section.10.4}
\contentsline {section}{\numberline {10.5}Example: Run-Length Coding Decompression in OpenMP}{228}{section.10.5}
\contentsline {section}{\numberline {10.6}Example: Run-Length Coding Decompression in Thrust}{229}{section.10.6}
\contentsline {section}{\numberline {10.7}Example: Moving Average}{230}{section.10.7}
\contentsline {subsection}{\numberline {10.7.1}Rth Code}{230}{subsection.10.7.1}
\contentsline {subsection}{\numberline {10.7.2}Algorithm}{232}{subsection.10.7.2}
\contentsline {subsection}{\numberline {10.7.3}Use of Lambda Functions}{232}{subsection.10.7.3}
\contentsline {chapter}{\numberline {11}Introduction to Parallel Matrix Operations}{235}{chapter.11}
\contentsline {section}{\numberline {11.1}``We're Not in Physicsland Anymore, Toto''}{235}{section.11.1}
\contentsline {section}{\numberline {11.2}Partitioned Matrices}{235}{section.11.2}
\contentsline {section}{\numberline {11.3}Parallel Matrix Multiplication}{237}{section.11.3}
\contentsline {subsection}{\numberline {11.3.1}Message-Passing Case}{237}{subsection.11.3.1}
\contentsline {subsubsection}{\numberline {11.3.1.1}Fox's Algorithm}{238}{subsubsection.11.3.1.1}
\contentsline {subsubsection}{\numberline {11.3.1.2}Performance Issues}{239}{subsubsection.11.3.1.2}
\contentsline {subsection}{\numberline {11.3.2}Shared-Memory Case}{239}{subsection.11.3.2}
\contentsline {subsubsection}{\numberline {11.3.2.1}Example: Matrix Multiply in OpenMP}{239}{subsubsection.11.3.2.1}
\contentsline {subsubsection}{\numberline {11.3.2.2}Example: Matrix Multiply in CUDA}{240}{subsubsection.11.3.2.2}
\contentsline {subsection}{\numberline {11.3.3}R Snow}{243}{subsection.11.3.3}
\contentsline {subsection}{\numberline {11.3.4}R Interfaces to GPUs}{243}{subsection.11.3.4}
\contentsline {section}{\numberline {11.4}Finding Powers of Matrices}{243}{section.11.4}
\contentsline {subsection}{\numberline {11.4.1}Example: Graph Connectedness}{243}{subsection.11.4.1}
\contentsline {subsection}{\numberline {11.4.2}Example: Fibonacci Numbers}{245}{subsection.11.4.2}
\contentsline {subsection}{\numberline {11.4.3}Example: Matrix Inversion}{246}{subsection.11.4.3}
\contentsline {subsection}{\numberline {11.4.4}Parallel Computation}{247}{subsection.11.4.4}
\contentsline {section}{\numberline {11.5}Solving Systems of Linear Equations}{247}{section.11.5}
\contentsline {subsection}{\numberline {11.5.1}Gaussian Elimination}{247}{subsection.11.5.1}
\contentsline {subsection}{\numberline {11.5.2}Example: Gaussian Elimination in CUDA}{248}{subsection.11.5.2}
\contentsline {subsection}{\numberline {11.5.3}The Jacobi Algorithm}{249}{subsection.11.5.3}
\contentsline {subsection}{\numberline {11.5.4}Example: OpenMP Implementation of the Jacobi Algorithm}{250}{subsection.11.5.4}
\contentsline {subsection}{\numberline {11.5.5}Example: R/gputools Implementation of Jacobi}{251}{subsection.11.5.5}
\contentsline {section}{\numberline {11.6}Eigenvalues and Eigenvectors}{252}{section.11.6}
\contentsline {subsection}{\numberline {11.6.1}The Power Method}{252}{subsection.11.6.1}
\contentsline {subsection}{\numberline {11.6.2}Parallel Computation}{253}{subsection.11.6.2}
\contentsline {section}{\numberline {11.7}Sparse Matrices}{253}{section.11.7}
\contentsline {section}{\numberline {11.8}Libraries}{255}{section.11.8}
\contentsline {chapter}{\numberline {12}Introduction to Parallel Sorting}{257}{chapter.12}
\contentsline {section}{\numberline {12.1}Quicksort}{257}{section.12.1}
\contentsline {subsection}{\numberline {12.1.1}The Separation Process}{257}{subsection.12.1.1}
\contentsline {subsection}{\numberline {12.1.2}Example: OpenMP Quicksort}{259}{subsection.12.1.2}
\contentsline {subsection}{\numberline {12.1.3}Hyperquicksort}{260}{subsection.12.1.3}
\contentsline {section}{\numberline {12.2}Mergesorts}{261}{section.12.2}
\contentsline {subsection}{\numberline {12.2.1}Sequential Form}{261}{subsection.12.2.1}
\contentsline {subsection}{\numberline {12.2.2}Shared-Memory Mergesort}{261}{subsection.12.2.2}
\contentsline {subsection}{\numberline {12.2.3}Message Passing Mergesort on a Tree Topology}{261}{subsection.12.2.3}
\contentsline {subsection}{\numberline {12.2.4}Compare-Exchange Operations}{262}{subsection.12.2.4}
\contentsline {subsection}{\numberline {12.2.5}Bitonic Mergesort}{262}{subsection.12.2.5}
\contentsline {section}{\numberline {12.3}The Bubble Sort and Its Cousins}{264}{section.12.3}
\contentsline {subsection}{\numberline {12.3.1}The Much-Maligned Bubble Sort}{264}{subsection.12.3.1}
\contentsline {subsection}{\numberline {12.3.2}A Popular Variant: Odd-Even Transposition}{265}{subsection.12.3.2}
\contentsline {subsection}{\numberline {12.3.3}Example: CUDA Implementation of Odd/Even Transposition Sort}{265}{subsection.12.3.3}
\contentsline {section}{\numberline {12.4}Shearsort}{267}{section.12.4}
\contentsline {section}{\numberline {12.5}Bucket Sort with Sampling}{267}{section.12.5}
\contentsline {section}{\numberline {12.6}Radix Sort}{271}{section.12.6}
\contentsline {section}{\numberline {12.7}Enumeration Sort}{271}{section.12.7}
\contentsline {chapter}{\numberline {13}Parallel Computation for Audio and Image Processing}{273}{chapter.13}
\contentsline {section}{\numberline {13.1}General Principles}{273}{section.13.1}
\contentsline {subsection}{\numberline {13.1.1}One-Dimensional Fourier Series}{273}{subsection.13.1.1}
\contentsline {subsection}{\numberline {13.1.2}Two-Dimensional Fourier Series}{277}{subsection.13.1.2}
\contentsline {section}{\numberline {13.2}Discrete Fourier Transforms}{277}{section.13.2}
\contentsline {subsection}{\numberline {13.2.1}One-Dimensional Data}{278}{subsection.13.2.1}
\contentsline {subsection}{\numberline {13.2.2}Inversion}{279}{subsection.13.2.2}
\contentsline {subsubsection}{\numberline {13.2.2.1}Alternate Formulation}{280}{subsubsection.13.2.2.1}
\contentsline {subsection}{\numberline {13.2.3}Two-Dimensional Data}{280}{subsection.13.2.3}
\contentsline {section}{\numberline {13.3}Parallel Computation of Discrete Fourier Transforms}{281}{section.13.3}
\contentsline {subsection}{\numberline {13.3.1}The Fast Fourier Transform}{281}{subsection.13.3.1}
\contentsline {subsection}{\numberline {13.3.2}A Matrix Approach}{282}{subsection.13.3.2}
\contentsline {subsection}{\numberline {13.3.3}Parallelizing Computation of the Inverse Transform}{282}{subsection.13.3.3}
\contentsline {subsection}{\numberline {13.3.4}Parallelizing Computation of the Two-Dimensional Transform}{282}{subsection.13.3.4}
\contentsline {section}{\numberline {13.4}Available FFT Software}{283}{section.13.4}
\contentsline {subsection}{\numberline {13.4.1}R}{283}{subsection.13.4.1}
\contentsline {subsection}{\numberline {13.4.2}CUFFT}{283}{subsection.13.4.2}
\contentsline {subsection}{\numberline {13.4.3}FFTW}{283}{subsection.13.4.3}
\contentsline {section}{\numberline {13.5}Applications to Image Processing}{284}{section.13.5}
\contentsline {subsection}{\numberline {13.5.1}Smoothing}{284}{subsection.13.5.1}
\contentsline {subsection}{\numberline {13.5.2}Example: Audio Smoothing in R}{284}{subsection.13.5.2}
\contentsline {subsection}{\numberline {13.5.3}Edge Detection}{285}{subsection.13.5.3}
\contentsline {section}{\numberline {13.6}R Access to Sound and Image Files}{286}{section.13.6}
\contentsline {section}{\numberline {13.7}Keeping the Pixel Intensities in the Proper Range}{286}{section.13.7}
\contentsline {section}{\numberline {13.8}Does the Function g() Really Have to Be Repeating?}{287}{section.13.8}
\contentsline {section}{\numberline {13.9}Vector Space Issues (optional section)}{287}{section.13.9}
\contentsline {section}{\numberline {13.10}Bandwidth: How to Read the \textit {San Francisco Chronicle} Business Page (optional section)}{289}{section.13.10}
\contentsline {chapter}{\numberline {14}Parallel Computation in Statistics/Data Mining}{291}{chapter.14}
\contentsline {section}{\numberline {14.1}Itemset Analysis}{291}{section.14.1}
\contentsline {subsection}{\numberline {14.1.1}What Is It?}{291}{subsection.14.1.1}
\contentsline {subsection}{\numberline {14.1.2}The Market Basket Problem}{292}{subsection.14.1.2}
\contentsline {subsection}{\numberline {14.1.3}Serial Algorithms}{293}{subsection.14.1.3}
\contentsline {subsection}{\numberline {14.1.4}Parallelizing the Apriori Algorithm}{294}{subsection.14.1.4}
\contentsline {section}{\numberline {14.2}Probability Density Estimation}{294}{section.14.2}
\contentsline {subsection}{\numberline {14.2.1}Kernel-Based Density Estimation}{295}{subsection.14.2.1}
\contentsline {subsection}{\numberline {14.2.2}Histogram Computation for Images}{298}{subsection.14.2.2}
\contentsline {section}{\numberline {14.3}Clustering}{299}{section.14.3}
\contentsline {subsection}{\numberline {14.3.1}Example: k-Means Clustering in R}{301}{subsection.14.3.1}
\contentsline {section}{\numberline {14.4}Principal Component Analysis (PCA)}{302}{section.14.4}
\contentsline {section}{\numberline {14.5}Monte Carlo Simulation}{303}{section.14.5}
\contentsline {chapter}{\numberline {A}Miscellaneous Systems Issues}{305}{appendix.A}
\contentsline {section}{\numberline {A.1}Timesharing}{305}{section.A.1}
\contentsline {subsection}{\numberline {A.1.1}Many Processes, Taking Turns}{305}{subsection.A.1.1}
\contentsline {section}{\numberline {A.2}Memory Hierarchies}{307}{section.A.2}
\contentsline {subsection}{\numberline {A.2.1}Cache Memory}{307}{subsection.A.2.1}
\contentsline {subsection}{\numberline {A.2.2}Virtual Memory}{307}{subsection.A.2.2}
\contentsline {subsubsection}{\numberline {A.2.2.1}Make Sure You Understand the Goals}{307}{subsubsection.A.2.2.1}
\contentsline {subsubsection}{\numberline {A.2.2.2}How It Works}{308}{subsubsection.A.2.2.2}
\contentsline {subsection}{\numberline {A.2.3}Performance Issues}{309}{subsection.A.2.3}
\contentsline {section}{\numberline {A.3}Array Issues}{309}{section.A.3}
\contentsline {subsection}{\numberline {A.3.1}Storage}{309}{subsection.A.3.1}
\contentsline {subsection}{\numberline {A.3.2}Subarrays}{310}{subsection.A.3.2}
\contentsline {subsection}{\numberline {A.3.3}Memory Allocation}{310}{subsection.A.3.3}
\contentsline {chapter}{\numberline {B}Review of Matrix Algebra}{313}{appendix.B}
\contentsline {section}{\numberline {B.1}Terminology and Notation}{313}{section.B.1}
\contentsline {subsection}{\numberline {B.1.1}Matrix Addition and Multiplication}{314}{subsection.B.1.1}
\contentsline {section}{\numberline {B.2}Matrix Transpose}{315}{section.B.2}
\contentsline {section}{\numberline {B.3}Linear Independence}{316}{section.B.3}
\contentsline {section}{\numberline {B.4}Determinants}{316}{section.B.4}
\contentsline {section}{\numberline {B.5}Matrix Inverse}{316}{section.B.5}
\contentsline {section}{\numberline {B.6}Eigenvalues and Eigenvectors}{317}{section.B.6}
\contentsline {section}{\numberline {B.7}Rank of a Matrix}{318}{section.B.7}
\contentsline {section}{\numberline {B.8}Matrix Algebra in R}{318}{section.B.8}
\contentsline {chapter}{\numberline {C}R Quick Start}{321}{appendix.C}
\contentsline {section}{\numberline {C.1}Correspondences}{321}{section.C.1}
\contentsline {section}{\numberline {C.2}Starting R}{322}{section.C.2}
\contentsline {section}{\numberline {C.3}First Sample Programming Session}{322}{section.C.3}
\contentsline {section}{\numberline {C.4}Vectorization}{324}{section.C.4}
\contentsline {section}{\numberline {C.5}Second Sample Programming Session}{325}{section.C.5}
\contentsline {section}{\numberline {C.6}Recycling}{326}{section.C.6}
\contentsline {section}{\numberline {C.7}More on Vectorization}{326}{section.C.7}
\contentsline {section}{\numberline {C.8}Third Sample Programming Session}{327}{section.C.8}
\contentsline {section}{\numberline {C.9}Default Argument Values}{328}{section.C.9}
\contentsline {section}{\numberline {C.10}The R List Type}{328}{section.C.10}
\contentsline {subsection}{\numberline {C.10.1}The Basics}{328}{subsection.C.10.1}
\contentsline {subsection}{\numberline {C.10.2}The Reduce() Function}{329}{subsection.C.10.2}
\contentsline {subsection}{\numberline {C.10.3}S3 Classes}{330}{subsection.C.10.3}
\contentsline {section}{\numberline {C.11}Some Workhorse Functions}{331}{section.C.11}
\contentsline {section}{\numberline {C.12}Handy Utilities}{332}{section.C.12}
\contentsline {section}{\numberline {C.13}Data Frames}{334}{section.C.13}
\contentsline {section}{\numberline {C.14}Graphics}{335}{section.C.14}
\contentsline {section}{\numberline {C.15}Packages}{336}{section.C.15}
\contentsline {section}{\numberline {C.16}Other Sources for Learning R}{336}{section.C.16}
\contentsline {section}{\numberline {C.17}Online Help}{337}{section.C.17}
\contentsline {section}{\numberline {C.18}Debugging in R}{337}{section.C.18}
\contentsline {section}{\numberline {C.19}Complex Numbers}{337}{section.C.19}
\contentsline {section}{\numberline {C.20}Further Reading}{338}{section.C.20}