\relax \providecommand\hyper@newdestlabel[2]{} \providecommand\HyperFirstAtBeginDocument{\AtBeginDocument} \HyperFirstAtBeginDocument{\ifx\hyper@anchor\@undefined \global\let\oldcontentsline\contentsline \gdef\contentsline#1#2#3#4{\oldcontentsline{#1}{#2}{#3}} \global\let\oldnewlabel\newlabel \gdef\newlabel#1#2{\newlabelxx{#1}#2} \gdef\newlabelxx#1#2#3#4#5#6{\oldnewlabel{#1}{{#2}{#3}}} \AtEndDocument{\ifx\hyper@anchor\@undefined \let\contentsline\oldcontentsline \let\newlabel\oldnewlabel \fi} \fi} \global\let\hyper@last\relax \gdef\HyperFirstAtBeginDocument#1{#1} \providecommand\HyField@AuxAddToFields[1]{} \providecommand\HyField@AuxAddToCoFields[2]{} \@writefile{toc}{\contentsline {chapter}{\numberline {1}Introduction to Parallel Processing}{1}{chapter.1}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \newlabel{chap:intro}{{1}{1}{Introduction to Parallel Processing}{chapter.1}{}} \@writefile{toc}{\contentsline {section}{\numberline {1.1}Platforms}{1}{section.1.1}} \@writefile{toc}{\contentsline {subsection}{\numberline {1.1.1}Why R?}{1}{subsection.1.1.1}} \@writefile{toc}{\contentsline {section}{\numberline {1.2}Why Use Parallel Systems?}{1}{section.1.2}} \@writefile{toc}{\contentsline {subsection}{\numberline {1.2.1}Execution Speed}{1}{subsection.1.2.1}} \@writefile{toc}{\contentsline {subsection}{\numberline {1.2.2}Memory}{2}{subsection.1.2.2}} \@writefile{toc}{\contentsline {subsection}{\numberline {1.2.3}Distributed Processing}{3}{subsection.1.2.3}} \@writefile{toc}{\contentsline {subsection}{\numberline {1.2.4}Our Focus Here}{3}{subsection.1.2.4}} \@writefile{toc}{\contentsline {section}{\numberline {1.3}Parallel Processing Hardware}{3}{section.1.3}} \@writefile{toc}{\contentsline {subsection}{\numberline {1.3.1}Shared-Memory Systems}{3}{subsection.1.3.1}} \@writefile{toc}{\contentsline {subsubsection}{\numberline {1.3.1.1}Basic Architecture}{3}{subsubsection.1.3.1.1}} \@writefile{toc}{\contentsline {subsubsection}{\numberline {1.3.1.2}Multiprocessor Topologies}{4}{subsubsection.1.3.1.2}} \@writefile{toc}{\contentsline {subsubsection}{\numberline {1.3.1.3}Memory Issues Etc.}{4}{subsubsection.1.3.1.3}} \@writefile{toc}{\contentsline {subsection}{\numberline {1.3.2}Message-Passing Systems}{5}{subsection.1.3.2}} \@writefile{toc}{\contentsline {subsubsection}{\numberline {1.3.2.1}Basic Architecture}{5}{subsubsection.1.3.2.1}} \@writefile{toc}{\contentsline {subsubsection}{\numberline {1.3.2.2}Example: Clusters}{5}{subsubsection.1.3.2.2}} \@writefile{toc}{\contentsline {subsection}{\numberline {1.3.3}SIMD}{5}{subsection.1.3.3}} \@writefile{toc}{\contentsline {section}{\numberline {1.4}Programmer World Views}{6}{section.1.4}} \@writefile{toc}{\contentsline {subsection}{\numberline {1.4.1}Example: Matrix-Vector Multiply}{6}{subsection.1.4.1}} \newlabel{matvec}{{1.4.1}{6}{Example: Matrix-Vector Multiply}{subsection.1.4.1}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {1.4.2}Shared-Memory}{6}{subsection.1.4.2}} \@writefile{toc}{\contentsline {subsubsection}{\numberline {1.4.2.1}Programmer View}{6}{subsubsection.1.4.2.1}} \newlabel{sharedview}{{1.4.2.1}{6}{Programmer View}{subsubsection.1.4.2.1}{}} \@writefile{toc}{\contentsline {section}{\numberline {1.5}Shared Memory Programming}{7}{section.1.5}} \newlabel{needknowos}{{1.5}{7}{Shared Memory Programming}{section.1.5}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {1.5.1}Low-Level Threads Systems: Pthreads}{7}{subsection.1.5.1}} \newlabel{pthreadsprime}{{1.5.1}{7}{Low-Level Threads Systems: Pthreads}{subsection.1.5.1}{}} \@writefile{toc}{\contentsline {subsubsection}{\numberline {1.5.1.1}Pthreads Example: Finding Primes}{8}{subsubsection.1.5.1.1}} \@writefile{toc}{\contentsline {subsection}{\numberline {1.5.2}Compiling and Linking}{10}{subsection.1.5.2}} \@writefile{toc}{\contentsline {subsection}{\numberline {1.5.3}The Notion of a ``Critical Section''}{10}{subsection.1.5.3}} \@writefile{toc}{\contentsline {subsection}{\numberline {1.5.4}Role of the OS}{13}{subsection.1.5.4}} \@writefile{toc}{\contentsline {subsection}{\numberline {1.5.5}Debugging Threads Programs}{13}{subsection.1.5.5}} \newlabel{debugthreads}{{1.5.5}{13}{Debugging Threads Programs}{subsection.1.5.5}{}} \@writefile{toc}{\contentsline {section}{\numberline {1.6}A Note on Global Variables}{14}{section.1.6}} \@writefile{toc}{\contentsline {subsection}{\numberline {1.6.1}Higher-Level Threads Programming: OpenMP}{15}{subsection.1.6.1}} \@writefile{toc}{\contentsline {subsubsection}{\numberline {1.6.1.1}Example: Sampling Bucket Sort}{15}{subsubsection.1.6.1.1}} \newlabel{ompbsort}{{1.6.1.1}{15}{Example: Sampling Bucket Sort}{subsubsection.1.6.1.1}{}} \@writefile{toc}{\contentsline {subsubsection}{\numberline {1.6.1.2}Debugging OpenMP}{18}{subsubsection.1.6.1.2}} \@writefile{toc}{\contentsline {subsection}{\numberline {1.6.2}Message Passing}{19}{subsection.1.6.2}} \@writefile{toc}{\contentsline {subsubsection}{\numberline {1.6.2.1}Programmer View}{19}{subsubsection.1.6.2.1}} \newlabel{msgview}{{1.6.2.1}{19}{Programmer View}{subsubsection.1.6.2.1}{}} \@writefile{toc}{\contentsline {subsubsection}{\numberline {1.6.2.2}Example: MPI Prime Numbers Finder}{19}{subsubsection.1.6.2.2}} \newlabel{mpiex}{{1.6.2.2}{19}{Example: MPI Prime Numbers Finder}{subsubsection.1.6.2.2}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {1.6.3}Scatter/Gather}{22}{subsection.1.6.3}} \@writefile{toc}{\contentsline {subsubsection}{\numberline {1.6.3.1}R snow Package}{23}{subsubsection.1.6.3.1}} \newlabel{snowintro}{{1.6.3.1}{23}{R snow Package}{subsubsection.1.6.3.1}{}} \@writefile{toc}{\contentsline {section}{\numberline {1.7}Threads Programming in R: Rdsm}{27}{section.1.7}} \@writefile{toc}{\contentsline {subsection}{\numberline {1.7.1}Example: Matrix Multiplication}{27}{subsection.1.7.1}} \@writefile{toc}{\contentsline {subsection}{\numberline {1.7.2}Example: Maximal Burst in a Time Series}{28}{subsection.1.7.2}} \@writefile{toc}{\contentsline {chapter}{\numberline {2}Recurring Performance Issues}{31}{chapter.2}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \newlabel{chap:issues}{{2}{31}{Recurring Performance Issues}{chapter.2}{}} \@writefile{toc}{\contentsline {section}{\numberline {2.1}Communication Bottlenecks}{31}{section.2.1}} \@writefile{toc}{\contentsline {section}{\numberline {2.2}Load Balancing}{32}{section.2.2}} \newlabel{loadbalance}{{2.2}{32}{Load Balancing}{section.2.2}{}} \@writefile{toc}{\contentsline {section}{\numberline {2.3}``Embarrassingly Parallel'' Applications}{32}{section.2.3}} \newlabel{embpar}{{2.3}{32}{``Embarrassingly Parallel'' Applications}{section.2.3}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {2.3.1}What People Mean by ``Embarrassingly Parallel''}{32}{subsection.2.3.1}} \@writefile{toc}{\contentsline {subsection}{\numberline {2.3.2}Iterative Algorithms}{33}{subsection.2.3.2}} \@writefile{toc}{\contentsline {section}{\numberline {2.4}Static (But Possibly Random) Task Assignment Typically Better Than Dynamic}{34}{section.2.4}} \newlabel{methodabest}{{2.4}{34}{Static (But Possibly Random) Task Assignment Typically Better Than Dynamic}{section.2.4}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {2.4.1}Example: Matrix-Vector Multiply}{34}{subsection.2.4.1}} \@writefile{toc}{\contentsline {subsection}{\numberline {2.4.2}Load Balance, Revisited}{36}{subsection.2.4.2}} \@writefile{toc}{\contentsline {subsection}{\numberline {2.4.3}Example: Mutual Web Outlinks}{37}{subsection.2.4.3}} \newlabel{mutlinks}{{2.4.3}{37}{Example: Mutual Web Outlinks}{subsection.2.4.3}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {2.4.4}Work Stealing}{38}{subsection.2.4.4}} \@writefile{toc}{\contentsline {subsection}{\numberline {2.4.5}Timing Example}{38}{subsection.2.4.5}} \newlabel{mandelbrottiming}{{2.4.5}{38}{Timing Example}{subsection.2.4.5}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {2.4.6}Example: Extracting a Submatrix}{38}{subsection.2.4.6}} \@writefile{toc}{\contentsline {section}{\numberline {2.5}Latency and Bandwidth}{39}{section.2.5}} \newlabel{latencybandwidth}{{2.5}{39}{Latency and Bandwidth}{section.2.5}{}} \@writefile{toc}{\contentsline {section}{\numberline {2.6}Relative Merits: Performance of Shared-Memory Vs. Message-Passing}{40}{section.2.6}} \newlabel{sharedbetter}{{2.6}{40}{Relative Merits: Performance of Shared-Memory Vs. Message-Passing}{section.2.6}{}} \@writefile{toc}{\contentsline {section}{\numberline {2.7}Memory Allocation Issues}{41}{section.2.7}} \@writefile{toc}{\contentsline {section}{\numberline {2.8}Issues Particular to Shared-Memory Systems}{41}{section.2.8}} \@writefile{toc}{\contentsline {chapter}{\numberline {3}Shared Memory Parallelism}{43}{chapter.3}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \newlabel{chap:shared}{{3}{43}{Shared Memory Parallelism}{chapter.3}{}} \@writefile{toc}{\contentsline {section}{\numberline {3.1}What Is Shared?}{43}{section.3.1}} \@writefile{toc}{\contentsline {section}{\numberline {3.2}Memory Modules}{44}{section.3.2}} \newlabel{banks}{{3.2}{44}{Memory Modules}{section.3.2}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {3.2.1}Interleaving}{44}{subsection.3.2.1}} \newlabel{interleaving}{{3.2.1}{44}{Interleaving}{subsection.3.2.1}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {3.2.2}Bank Conflicts and Solutions}{45}{subsection.3.2.2}} \newlabel{bankclash}{{3.2.2}{45}{Bank Conflicts and Solutions}{subsection.3.2.2}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {3.2.3}Example: Code to Implement Padding}{47}{subsection.3.2.3}} \@writefile{toc}{\contentsline {section}{\numberline {3.3}Interconnection Topologies}{48}{section.3.3}} \@writefile{toc}{\contentsline {subsection}{\numberline {3.3.1}SMP Systems}{48}{subsection.3.3.1}} \@writefile{toc}{\contentsline {subsection}{\numberline {3.3.2}NUMA Systems}{49}{subsection.3.3.2}} \@writefile{toc}{\contentsline {subsection}{\numberline {3.3.3}NUMA Interconnect Topologies}{50}{subsection.3.3.3}} \@writefile{toc}{\contentsline {subsubsection}{\numberline {3.3.3.1}Crossbar Interconnects}{50}{subsubsection.3.3.3.1}} \@writefile{toc}{\contentsline {subsubsection}{\numberline {3.3.3.2}Omega (or Delta) Interconnects}{52}{subsubsection.3.3.3.2}} \@writefile{toc}{\contentsline {subsection}{\numberline {3.3.4}Comparative Analysis}{53}{subsection.3.3.4}} \@writefile{toc}{\contentsline {subsection}{\numberline {3.3.5}Why Have Memory in Modules?}{54}{subsection.3.3.5}} \@writefile{toc}{\contentsline {section}{\numberline {3.4}Synchronization Hardware}{55}{section.3.4}} \@writefile{toc}{\contentsline {subsection}{\numberline {3.4.1}Test-and-Set Instructions}{55}{subsection.3.4.1}} \@writefile{toc}{\contentsline {subsubsection}{\numberline {3.4.1.1}LOCK Prefix on Intel Processors}{55}{subsubsection.3.4.1.1}} \newlabel{lockprefix}{{3.4.1.1}{55}{LOCK Prefix on Intel Processors}{subsubsection.3.4.1.1}{}} \@writefile{toc}{\contentsline {subsubsection}{\numberline {3.4.1.2}Locks with More Complex Interconnects}{57}{subsubsection.3.4.1.2}} \@writefile{toc}{\contentsline {subsection}{\numberline {3.4.2}May Not Need the Latest}{57}{subsection.3.4.2}} \@writefile{toc}{\contentsline {subsection}{\numberline {3.4.3}Fetch-and-Add Instructions}{57}{subsection.3.4.3}} \newlabel{fanda}{{3.4.3}{57}{Fetch-and-Add Instructions}{subsection.3.4.3}{}} \@writefile{toc}{\contentsline {section}{\numberline {3.5}Cache Issues}{58}{section.3.5}} \newlabel{cacheissues}{{3.5}{58}{Cache Issues}{section.3.5}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {3.5.1}Cache Coherency}{58}{subsection.3.5.1}} \newlabel{cachecoherency}{{3.5.1}{58}{Cache Coherency}{subsection.3.5.1}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {3.5.2}Example: the MESI Cache Coherency Protocol}{61}{subsection.3.5.2}} \@writefile{toc}{\contentsline {subsection}{\numberline {3.5.3}The Problem of ``False Sharing''}{63}{subsection.3.5.3}} \@writefile{toc}{\contentsline {section}{\numberline {3.6}Memory-Access Consistency Policies}{64}{section.3.6}} \newlabel{consistency}{{3.6}{64}{Memory-Access Consistency Policies}{section.3.6}{}} \@writefile{toc}{\contentsline {section}{\numberline {3.7}Fetch-and-Add Combining within Interconnects}{66}{section.3.7}} \@writefile{toc}{\contentsline {section}{\numberline {3.8}Multicore Chips}{66}{section.3.8}} \@writefile{toc}{\contentsline {section}{\numberline {3.9}Optimal Number of Threads}{67}{section.3.9}} \@writefile{toc}{\contentsline {section}{\numberline {3.10}Processor Affinity}{67}{section.3.10}} \@writefile{toc}{\contentsline {section}{\numberline {3.11}Illusion of Shared-Memory through Software}{68}{section.3.11}} \newlabel{sdsm}{{3.11}{68}{Illusion of Shared-Memory through Software}{section.3.11}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {3.11.1}Software Distributed Shared Memory}{68}{subsection.3.11.1}} \@writefile{toc}{\contentsline {subsection}{\numberline {3.11.2}Case Study: JIAJIA}{70}{subsection.3.11.2}} \@writefile{toc}{\contentsline {section}{\numberline {3.12}Barrier Implementation}{73}{section.3.12}} \@writefile{toc}{\contentsline {subsection}{\numberline {3.12.1}A Use-Once Version}{74}{subsection.3.12.1}} \@writefile{toc}{\contentsline {subsection}{\numberline {3.12.2}An Attempt to Write a Reusable Version}{75}{subsection.3.12.2}} \@writefile{toc}{\contentsline {subsection}{\numberline {3.12.3}A Correct Version}{75}{subsection.3.12.3}} \@writefile{toc}{\contentsline {subsection}{\numberline {3.12.4}Refinements}{76}{subsection.3.12.4}} \@writefile{toc}{\contentsline {subsubsection}{\numberline {3.12.4.1}Use of Wait Operations}{76}{subsubsection.3.12.4.1}} \@writefile{toc}{\contentsline {subsubsection}{\numberline {3.12.4.2}Parallelizing the Barrier Operation}{78}{subsubsection.3.12.4.2}} \@writefile{toc}{\contentsline {paragraph}{\numberline {3.12.4.2.1}Tree Barriers}{78}{paragraph.3.12.4.2.1}} \@writefile{toc}{\contentsline {paragraph}{\numberline {3.12.4.2.2}Butterfly Barriers}{78}{paragraph.3.12.4.2.2}} \@writefile{toc}{\contentsline {chapter}{\numberline {4}Introduction to OpenMP}{81}{chapter.4}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \newlabel{chap:omp}{{4}{81}{Introduction to OpenMP}{chapter.4}{}} \@writefile{toc}{\contentsline {section}{\numberline {4.1}Overview}{81}{section.4.1}} \@writefile{toc}{\contentsline {section}{\numberline {4.2}Example: Dijkstra Shortest-Path Algorithm}{81}{section.4.2}} \newlabel{running}{{4.2}{81}{Example: Dijkstra Shortest-Path Algorithm}{section.4.2}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {4.2.1}The Algorithm}{84}{subsection.4.2.1}} \@writefile{toc}{\contentsline {subsection}{\numberline {4.2.2}The OpenMP {\tt parallel} Pragma}{84}{subsection.4.2.2}} \@writefile{toc}{\contentsline {subsection}{\numberline {4.2.3}Scope Issues}{85}{subsection.4.2.3}} \@writefile{toc}{\contentsline {subsection}{\numberline {4.2.4}The OpenMP {\tt single} Pragma}{86}{subsection.4.2.4}} \@writefile{toc}{\contentsline {subsection}{\numberline {4.2.5}The OpenMP {\tt barrier} Pragma}{86}{subsection.4.2.5}} \@writefile{toc}{\contentsline {subsection}{\numberline {4.2.6}Implicit Barriers}{87}{subsection.4.2.6}} \@writefile{toc}{\contentsline {subsection}{\numberline {4.2.7}The OpenMP {\tt critical} Pragma}{87}{subsection.4.2.7}} \@writefile{toc}{\contentsline {section}{\numberline {4.3}The OpenMP {\tt for} Pragma}{87}{section.4.3}} \@writefile{toc}{\contentsline {subsection}{\numberline {4.3.1}Example: Dijkstra with Parallel for Loops}{87}{subsection.4.3.1}} \newlabel{basic}{{4.3.1}{87}{Example: Dijkstra with Parallel for Loops}{subsection.4.3.1}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {4.3.2}Nested Loops}{90}{subsection.4.3.2}} \@writefile{toc}{\contentsline {subsection}{\numberline {4.3.3}Controlling the Partitioning of Work to Threads: the schedule Clause}{90}{subsection.4.3.3}} \newlabel{schedulework}{{4.3.3}{90}{Controlling the Partitioning of Work to Threads: the schedule Clause}{subsection.4.3.3}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {4.3.4}Example: In-Place Matrix Transpose}{92}{subsection.4.3.4}} \@writefile{toc}{\contentsline {subsection}{\numberline {4.3.5}The OpenMP {\tt reduction} Clause}{93}{subsection.4.3.5}} \newlabel{ompreduction}{{4.3.5}{93}{The OpenMP {\tt reduction} Clause}{subsection.4.3.5}{}} \@writefile{toc}{\contentsline {section}{\numberline {4.4}Example: Mandelbrot Set}{94}{section.4.4}} \newlabel{mandelbrotcode}{{4.4}{94}{Example: Mandelbrot Set}{section.4.4}{}} \@writefile{toc}{\contentsline {section}{\numberline {4.5}The Task Directive}{97}{section.4.5}} \newlabel{taskdir}{{4.5}{97}{The Task Directive}{section.4.5}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {4.5.1}Example: Quicksort}{98}{subsection.4.5.1}} \@writefile{toc}{\contentsline {section}{\numberline {4.6}Other OpenMP Synchronization Issues}{99}{section.4.6}} \@writefile{toc}{\contentsline {subsection}{\numberline {4.6.1}The OpenMP {\tt atomic} Clause}{99}{subsection.4.6.1}} \@writefile{toc}{\contentsline {subsection}{\numberline {4.6.2}Memory Consistency and the {\tt flush} Pragma}{100}{subsection.4.6.2}} \@writefile{toc}{\contentsline {section}{\numberline {4.7}Combining Work-Sharing Constructs}{101}{section.4.7}} \@writefile{toc}{\contentsline {section}{\numberline {4.8}The Rest of OpenMP}{102}{section.4.8}} \@writefile{toc}{\contentsline {section}{\numberline {4.9}Compiling, Running and Debugging OpenMP Code}{102}{section.4.9}} \@writefile{toc}{\contentsline {subsection}{\numberline {4.9.1}Compiling}{102}{subsection.4.9.1}} \@writefile{toc}{\contentsline {subsection}{\numberline {4.9.2}Running}{103}{subsection.4.9.2}} \@writefile{toc}{\contentsline {subsection}{\numberline {4.9.3}Debugging}{103}{subsection.4.9.3}} \@writefile{toc}{\contentsline {section}{\numberline {4.10}Performance}{104}{section.4.10}} \@writefile{toc}{\contentsline {subsection}{\numberline {4.10.1}The Effect of Problem Size}{104}{subsection.4.10.1}} \@writefile{toc}{\contentsline {subsection}{\numberline {4.10.2}Some Fine Tuning}{104}{subsection.4.10.2}} \@writefile{toc}{\contentsline {subsection}{\numberline {4.10.3}OpenMP Internals}{108}{subsection.4.10.3}} \@writefile{toc}{\contentsline {section}{\numberline {4.11}Example: Root Finding}{109}{section.4.11}} \@writefile{toc}{\contentsline {section}{\numberline {4.12}Example: Mutual Outlinks}{110}{section.4.12}} \newlabel{ompmutlinks}{{4.12}{110}{Example: Mutual Outlinks}{section.4.12}{}} \@writefile{toc}{\contentsline {section}{\numberline {4.13}Example: Transforming an Adjacency Matrix}{112}{section.4.13}} \newlabel{transgraph}{{4.13}{112}{Example: Transforming an Adjacency Matrix}{section.4.13}{}} \@writefile{toc}{\contentsline {section}{\numberline {4.14}Example: Finding the Maximal Burst in a Time Series}{115}{section.4.14}} \@writefile{toc}{\contentsline {section}{\numberline {4.15}Locks with OpenMP}{117}{section.4.15}} \@writefile{toc}{\contentsline {section}{\numberline {4.16}Other Examples of OpenMP Code in This Book}{117}{section.4.16}} \@writefile{toc}{\contentsline {chapter}{\numberline {5}Introduction to GPU Programming with CUDA}{119}{chapter.5}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \newlabel{chap:cuda}{{5}{119}{Introduction to GPU Programming with CUDA}{chapter.5}{}} \@writefile{toc}{\contentsline {section}{\numberline {5.1}Overview}{119}{section.5.1}} \@writefile{toc}{\contentsline {section}{\numberline {5.2}Some Terminology}{120}{section.5.2}} \@writefile{toc}{\contentsline {section}{\numberline {5.3}Example: Calculate Row Sums}{120}{section.5.3}} \@writefile{toc}{\contentsline {section}{\numberline {5.4}Understanding the Hardware Structure}{124}{section.5.4}} \@writefile{toc}{\contentsline {subsection}{\numberline {5.4.1}Processing Units}{124}{subsection.5.4.1}} \@writefile{toc}{\contentsline {subsection}{\numberline {5.4.2}Thread Operation}{125}{subsection.5.4.2}} \@writefile{toc}{\contentsline {subsubsection}{\numberline {5.4.2.1}SIMT Architecture}{125}{subsubsection.5.4.2.1}} \@writefile{toc}{\contentsline {subsubsection}{\numberline {5.4.2.2}The Problem of Thread Divergence}{125}{subsubsection.5.4.2.2}} \@writefile{toc}{\contentsline {subsubsection}{\numberline {5.4.2.3}``OS in Hardware''}{125}{subsubsection.5.4.2.3}} \@writefile{toc}{\contentsline {subsection}{\numberline {5.4.3}Memory Structure}{126}{subsection.5.4.3}} \@writefile{toc}{\contentsline {subsubsection}{\numberline {5.4.3.1}Shared and Global Memory}{126}{subsubsection.5.4.3.1}} \@writefile{toc}{\contentsline {subsubsection}{\numberline {5.4.3.2}Global-Memory Performance Issues}{130}{subsubsection.5.4.3.2}} \newlabel{coal}{{5.4.3.2}{130}{Global-Memory Performance Issues}{subsubsection.5.4.3.2}{}} \@writefile{toc}{\contentsline {subsubsection}{\numberline {5.4.3.3}Shared-Memory Performance Issues}{130}{subsubsection.5.4.3.3}} \@writefile{toc}{\contentsline {subsubsection}{\numberline {5.4.3.4}Host/Device Memory Transfer Performance Issues}{131}{subsubsection.5.4.3.4}} \@writefile{toc}{\contentsline {subsubsection}{\numberline {5.4.3.5}Other Types of Memory}{131}{subsubsection.5.4.3.5}} \newlabel{othermems}{{5.4.3.5}{131}{Other Types of Memory}{subsubsection.5.4.3.5}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {5.4.4}Threads Hierarchy}{133}{subsection.5.4.4}} \newlabel{threadshierarchy}{{5.4.4}{133}{Threads Hierarchy}{subsection.5.4.4}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {5.4.5}What's NOT There}{135}{subsection.5.4.5}} \@writefile{toc}{\contentsline {section}{\numberline {5.5}Synchronization, Within and Between Blocks}{135}{section.5.5}} \@writefile{toc}{\contentsline {section}{\numberline {5.6}More on the Blocks/Threads Tradeoff}{136}{section.5.6}} \@writefile{toc}{\contentsline {section}{\numberline {5.7}Hardware Requirements, Installation, Compilation, Debugging}{137}{section.5.7}} \@writefile{toc}{\contentsline {section}{\numberline {5.8}Example: Improving the Row Sums Program}{139}{section.5.8}} \@writefile{toc}{\contentsline {section}{\numberline {5.9}Example: Finding the Mean Number of Mutual Outlinks}{141}{section.5.9}} \@writefile{toc}{\contentsline {section}{\numberline {5.10}Example: Finding Prime Numbers}{142}{section.5.10}} \newlabel{primefinder}{{5.10}{142}{Example: Finding Prime Numbers}{section.5.10}{}} \@writefile{toc}{\contentsline {section}{\numberline {5.11}Example: Finding Cumulative Sums}{145}{section.5.11}} \newlabel{cumulsums}{{5.11}{145}{Example: Finding Cumulative Sums}{section.5.11}{}} \@writefile{toc}{\contentsline {section}{\numberline {5.12}When Is It Advantageous to Use Shared Memory}{147}{section.5.12}} \@writefile{toc}{\contentsline {section}{\numberline {5.13}Example: Transforming an Adjacency Matrix}{147}{section.5.13}} \newlabel{transgraph1}{{5.13}{147}{Example: Transforming an Adjacency Matrix}{section.5.13}{}} \@writefile{toc}{\contentsline {section}{\numberline {5.14}Error Checking}{150}{section.5.14}} \@writefile{toc}{\contentsline {section}{\numberline {5.15}Loop Unrolling}{151}{section.5.15}} \@writefile{toc}{\contentsline {section}{\numberline {5.16}Short Vectors}{152}{section.5.16}} \@writefile{toc}{\contentsline {section}{\numberline {5.17}Newer Generations}{152}{section.5.17}} \newlabel{newergens}{{5.17}{152}{Newer Generations}{section.5.17}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {5.17.1}True Caching}{152}{subsection.5.17.1}} \@writefile{toc}{\contentsline {subsection}{\numberline {5.17.2}Unified Memory}{152}{subsection.5.17.2}} \@writefile{toc}{\contentsline {section}{\numberline {5.18}CUDA from a Higher Level}{153}{section.5.18}} \@writefile{toc}{\contentsline {subsection}{\numberline {5.18.1}CUBLAS}{153}{subsection.5.18.1}} \newlabel{cublas}{{5.18.1}{153}{CUBLAS}{subsection.5.18.1}{}} \@writefile{toc}{\contentsline {subsubsection}{\numberline {5.18.1.1}Example: Row Sums Once Again}{153}{subsubsection.5.18.1.1}} \@writefile{toc}{\contentsline {subsection}{\numberline {5.18.2}Thrust}{155}{subsection.5.18.2}} \@writefile{toc}{\contentsline {subsection}{\numberline {5.18.3}CUFFT}{156}{subsection.5.18.3}} \@writefile{toc}{\contentsline {section}{\numberline {5.19}Other CUDA Examples in This Book}{156}{section.5.19}} \@writefile{toc}{\contentsline {chapter}{\numberline {6}Introduction to Thrust Programming}{157}{chapter.6}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \newlabel{chap:thrust}{{6}{157}{Introduction to Thrust Programming}{chapter.6}{}} \@writefile{toc}{\contentsline {section}{\numberline {6.1}Compiling Thrust Code}{157}{section.6.1}} \@writefile{toc}{\contentsline {subsection}{\numberline {6.1.1}Compiling to CUDA}{157}{subsection.6.1.1}} \@writefile{toc}{\contentsline {subsection}{\numberline {6.1.2}Compiling to OpenMP}{158}{subsection.6.1.2}} \@writefile{toc}{\contentsline {section}{\numberline {6.2}Example: Counting the Number of Unique Values in an Array}{158}{section.6.2}} \@writefile{toc}{\contentsline {section}{\numberline {6.3}Example: A Plain-C Wrapper for Thrust sort()}{162}{section.6.3}} \@writefile{toc}{\contentsline {section}{\numberline {6.4}Example: Calculating Percentiles in an Array}{163}{section.6.4}} \@writefile{toc}{\contentsline {section}{\numberline {6.5}Mixing Thrust and CUDA Code}{165}{section.6.5}} \@writefile{toc}{\contentsline {section}{\numberline {6.6}Example: Doubling Every k$^{th}$ Element of an Array}{166}{section.6.6}} \newlabel{doubling}{{6.6}{166}{Example: Doubling Every k$^{th}$ Element of an Array}{section.6.6}{}} \@writefile{toc}{\contentsline {section}{\numberline {6.7}Scatter and Gather Operations}{167}{section.6.7}} \@writefile{toc}{\contentsline {subsection}{\numberline {6.7.1}Example: Matrix Transpose}{169}{subsection.6.7.1}} \newlabel{matxpose}{{6.7.1}{169}{Example: Matrix Transpose}{subsection.6.7.1}{}} \@writefile{toc}{\contentsline {section}{\numberline {6.8}Advanced (``Fancy'') Iterators}{170}{section.6.8}} \@writefile{toc}{\contentsline {subsection}{\numberline {6.8.1}Example: Matrix Transpose Again}{170}{subsection.6.8.1}} \@writefile{toc}{\contentsline {section}{\numberline {6.9}A Timing Comparison}{172}{section.6.9}} \@writefile{toc}{\contentsline {section}{\numberline {6.10}Example: Transforming an Adjacency Matrix}{176}{section.6.10}} \@writefile{toc}{\contentsline {section}{\numberline {6.11}Prefix Scan}{178}{section.6.11}} \@writefile{toc}{\contentsline {section}{\numberline {6.12}More on Use of Thrust for a CUDA Backend}{179}{section.6.12}} \@writefile{toc}{\contentsline {subsection}{\numberline {6.12.1}Synchronicity}{179}{subsection.6.12.1}} \@writefile{toc}{\contentsline {section}{\numberline {6.13}Error Messages}{179}{section.6.13}} \@writefile{toc}{\contentsline {section}{\numberline {6.14}Other Examples of Thrust Code in This Book}{180}{section.6.14}} \@writefile{toc}{\contentsline {chapter}{\numberline {7}Message Passing Systems}{181}{chapter.7}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \newlabel{chap:msg}{{7}{181}{Message Passing Systems}{chapter.7}{}} \@writefile{toc}{\contentsline {section}{\numberline {7.1}Overview}{181}{section.7.1}} \@writefile{toc}{\contentsline {section}{\numberline {7.2}A Historical Example: Hypercubes}{182}{section.7.2}} \@writefile{toc}{\contentsline {subsection}{\numberline {7.2.1}Definitions}{182}{subsection.7.2.1}} \@writefile{toc}{\contentsline {section}{\numberline {7.3}Networks of Workstations (NOWs)}{184}{section.7.3}} \@writefile{toc}{\contentsline {subsection}{\numberline {7.3.1}The Network Is Literally the Weakest Link}{184}{subsection.7.3.1}} \@writefile{toc}{\contentsline {subsection}{\numberline {7.3.2}Other Issues}{185}{subsection.7.3.2}} \@writefile{toc}{\contentsline {section}{\numberline {7.4}Scatter/Gather Operations}{185}{section.7.4}} \newlabel{scattergather}{{7.4}{185}{Scatter/Gather Operations}{section.7.4}{}} \@writefile{toc}{\contentsline {chapter}{\numberline {8}Introduction to MPI}{187}{chapter.8}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \newlabel{chap:mpi}{{8}{187}{Introduction to MPI}{chapter.8}{}} \@writefile{toc}{\contentsline {section}{\numberline {8.1}Overview}{187}{section.8.1}} \@writefile{toc}{\contentsline {subsection}{\numberline {8.1.1}History}{187}{subsection.8.1.1}} \@writefile{toc}{\contentsline {subsection}{\numberline {8.1.2}Structure and Execution}{188}{subsection.8.1.2}} \@writefile{toc}{\contentsline {subsection}{\numberline {8.1.3}Implementations}{188}{subsection.8.1.3}} \@writefile{toc}{\contentsline {subsection}{\numberline {8.1.4}Performance Issues}{188}{subsection.8.1.4}} \@writefile{toc}{\contentsline {section}{\numberline {8.2}Review of Earlier Example}{189}{section.8.2}} \@writefile{toc}{\contentsline {section}{\numberline {8.3}Example: Dijkstra Algorithm}{189}{section.8.3}} \@writefile{toc}{\contentsline {subsection}{\numberline {8.3.1}The Algorithm}{189}{subsection.8.3.1}} \@writefile{toc}{\contentsline {subsection}{\numberline {8.3.2}The MPI Code}{190}{subsection.8.3.2}} \@writefile{toc}{\contentsline {subsection}{\numberline {8.3.3}Introduction to MPI APIs}{193}{subsection.8.3.3}} \@writefile{toc}{\contentsline {subsubsection}{\numberline {8.3.3.1}MPI\_Init() and MPI\_Finalize()}{193}{subsubsection.8.3.3.1}} \@writefile{toc}{\contentsline {subsubsection}{\numberline {8.3.3.2}MPI\_Comm\_size() and MPI\_Comm\_rank()}{193}{subsubsection.8.3.3.2}} \@writefile{toc}{\contentsline {subsubsection}{\numberline {8.3.3.3}MPI\_Send()}{194}{subsubsection.8.3.3.3}} \@writefile{toc}{\contentsline {subsubsection}{\numberline {8.3.3.4}MPI\_Recv()}{195}{subsubsection.8.3.3.4}} \@writefile{toc}{\contentsline {section}{\numberline {8.4}Example: Removing 0s from an Array}{196}{section.8.4}} \@writefile{toc}{\contentsline {section}{\numberline {8.5}Debugging MPI Code}{197}{section.8.5}} \@writefile{toc}{\contentsline {section}{\numberline {8.6}Collective Communications}{198}{section.8.6}} \@writefile{toc}{\contentsline {subsection}{\numberline {8.6.1}Example: Refined Dijkstra Code}{198}{subsection.8.6.1}} \@writefile{toc}{\contentsline {subsection}{\numberline {8.6.2}MPI\_Bcast()}{201}{subsection.8.6.2}} \@writefile{toc}{\contentsline {subsection}{\numberline {8.6.3}MPI\_Reduce()/MPI\_Allreduce()}{202}{subsection.8.6.3}} \newlabel{mpireduction}{{8.6.3}{202}{MPI\_Reduce()/MPI\_Allreduce()}{subsection.8.6.3}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {8.6.4}MPI\_Gather()/MPI\_Allgather()}{203}{subsection.8.6.4}} \@writefile{toc}{\contentsline {subsection}{\numberline {8.6.5}The MPI\_Scatter()}{203}{subsection.8.6.5}} \@writefile{toc}{\contentsline {subsection}{\numberline {8.6.6}Example: Count the Number of Edges in a Directed Graph}{204}{subsection.8.6.6}} \@writefile{toc}{\contentsline {subsection}{\numberline {8.6.7}Example: Cumulative Sums}{204}{subsection.8.6.7}} \@writefile{toc}{\contentsline {subsection}{\numberline {8.6.8}Example: an MPI Solution to the Mutual Outlinks Problem}{206}{subsection.8.6.8}} \@writefile{toc}{\contentsline {subsection}{\numberline {8.6.9}The MPI\_Barrier()}{208}{subsection.8.6.9}} \@writefile{toc}{\contentsline {subsection}{\numberline {8.6.10}Creating Communicators}{208}{subsection.8.6.10}} \@writefile{toc}{\contentsline {section}{\numberline {8.7}Buffering, Synchrony and Related Issues}{209}{section.8.7}} \@writefile{toc}{\contentsline {subsection}{\numberline {8.7.1}Buffering, Etc.}{209}{subsection.8.7.1}} \@writefile{toc}{\contentsline {subsection}{\numberline {8.7.2}Safety}{210}{subsection.8.7.2}} \@writefile{toc}{\contentsline {subsection}{\numberline {8.7.3}Living Dangerously}{211}{subsection.8.7.3}} \@writefile{toc}{\contentsline {subsection}{\numberline {8.7.4}Safe Exchange Operations}{211}{subsection.8.7.4}} \@writefile{toc}{\contentsline {section}{\numberline {8.8}Use of MPI from Other Languages}{211}{section.8.8}} \@writefile{toc}{\contentsline {section}{\numberline {8.9}Other MPI Examples in This Book}{212}{section.8.9}} \@writefile{toc}{\contentsline {chapter}{\numberline {9}MapReduce Computation}{213}{chapter.9}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \newlabel{chap:mapreduce}{{9}{213}{MapReduce Computation}{chapter.9}{}} \@writefile{toc}{\contentsline {section}{\numberline {9.1}Apache Hadoop}{214}{section.9.1}} \@writefile{toc}{\contentsline {subsection}{\numberline {9.1.1}Hadoop Streaming}{214}{subsection.9.1.1}} \@writefile{toc}{\contentsline {subsection}{\numberline {9.1.2}Example: Word Count}{214}{subsection.9.1.2}} \newlabel{hadoopwordcount}{{9.1.2}{214}{Example: Word Count}{subsection.9.1.2}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {9.1.3}Running the Code}{215}{subsection.9.1.3}} \@writefile{toc}{\contentsline {subsection}{\numberline {9.1.4}Analysis of the Code}{216}{subsection.9.1.4}} \@writefile{toc}{\contentsline {subsection}{\numberline {9.1.5}Role of Disk Files}{217}{subsection.9.1.5}} \@writefile{toc}{\contentsline {section}{\numberline {9.2}Other MapReduce Systems}{218}{section.9.2}} \@writefile{toc}{\contentsline {section}{\numberline {9.3}R Interfaces to MapReduce Systems}{218}{section.9.3}} \@writefile{toc}{\contentsline {section}{\numberline {9.4}An Alternative: ``Snowdoop''}{218}{section.9.4}} \newlabel{snowdoop}{{9.4}{218}{An Alternative: ``Snowdoop''}{section.9.4}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {9.4.1}Example: Snowdoop Word Count}{219}{subsection.9.4.1}} \@writefile{toc}{\contentsline {subsection}{\numberline {9.4.2}Example: Snowdoop k-Means Clustering}{220}{subsection.9.4.2}} \@writefile{toc}{\contentsline {chapter}{\numberline {10}The Parallel Prefix Problem}{223}{chapter.10}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \newlabel{chap:prefix}{{10}{223}{The Parallel Prefix Problem}{chapter.10}{}} \newlabel{scandef}{{10.1}{223}{The Parallel Prefix Problem}{equation.10.0.1}{}} \@writefile{toc}{\contentsline {section}{\numberline {10.1}Example: Permutations}{223}{section.10.1}} \newlabel{fibprefix}{{10.1}{223}{Example: Permutations}{section.10.1}{}} \@writefile{toc}{\contentsline {section}{\numberline {10.2}General Strategies for Parallel Scan Computation}{224}{section.10.2}} \newlabel{genpar}{{10.2}{224}{General Strategies for Parallel Scan Computation}{section.10.2}{}} \@writefile{toc}{\contentsline {section}{\numberline {10.3}Implementations}{227}{section.10.3}} \newlabel{prefiximps}{{10.3}{227}{Implementations}{section.10.3}{}} \@writefile{toc}{\contentsline {section}{\numberline {10.4}Example: Parallel Prefix Summing in OpenMP}{227}{section.10.4}} \@writefile{toc}{\contentsline {section}{\numberline {10.5}Example: Run-Length Coding Decompression in OpenMP}{228}{section.10.5}} \@writefile{toc}{\contentsline {section}{\numberline {10.6}Example: Run-Length Coding Decompression in Thrust}{229}{section.10.6}} \newlabel{thrustrunlength}{{10.6}{229}{Example: Run-Length Coding Decompression in Thrust}{section.10.6}{}} \@writefile{toc}{\contentsline {section}{\numberline {10.7}Example: Moving Average}{230}{section.10.7}} \newlabel{ma}{{10.7}{230}{Example: Moving Average}{section.10.7}{}} \newlabel{defineai}{{10.21}{230}{Example: Moving Average}{equation.10.7.21}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {10.7.1}Rth Code}{230}{subsection.10.7.1}} \@writefile{toc}{\contentsline {subsection}{\numberline {10.7.2}Algorithm}{232}{subsection.10.7.2}} \newlabel{ainumer}{{10.22}{232}{Algorithm}{equation.10.7.22}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {10.7.3}Use of Lambda Functions}{232}{subsection.10.7.3}} \newlabel{lambda}{{10.7.3}{232}{Use of Lambda Functions}{subsection.10.7.3}{}} \@writefile{toc}{\contentsline {chapter}{\numberline {11}Introduction to Parallel Matrix Operations}{235}{chapter.11}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \newlabel{chap:matrix}{{11}{235}{Introduction to Parallel Matrix Operations}{chapter.11}{}} \@writefile{toc}{\contentsline {section}{\numberline {11.1}``We're Not in Physicsland Anymore, Toto''}{235}{section.11.1}} \newlabel{notallphys}{{11.1}{235}{``We're Not in Physicsland Anymore, Toto''}{section.11.1}{}} \@writefile{toc}{\contentsline {section}{\numberline {11.2}Partitioned Matrices}{235}{section.11.2}} \newlabel{partitioned}{{11.2}{235}{Partitioned Matrices}{section.11.2}{}} \@writefile{toc}{\contentsline {section}{\numberline {11.3}Parallel Matrix Multiplication}{237}{section.11.3}} \newlabel{parmatmult}{{11.3}{237}{Parallel Matrix Multiplication}{section.11.3}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {11.3.1}Message-Passing Case}{237}{subsection.11.3.1}} \@writefile{toc}{\contentsline {subsubsection}{\numberline {11.3.1.1}Fox's Algorithm}{238}{subsubsection.11.3.1.1}} \newlabel{matmult}{{11.3.1.1}{238}{Fox's Algorithm}{subsubsection.11.3.1.1}{}} \@writefile{toc}{\contentsline {subsubsection}{\numberline {11.3.1.2}Performance Issues}{239}{subsubsection.11.3.1.2}} \@writefile{toc}{\contentsline {subsection}{\numberline {11.3.2}Shared-Memory Case}{239}{subsection.11.3.2}} \@writefile{toc}{\contentsline {subsubsection}{\numberline {11.3.2.1}Example: Matrix Multiply in OpenMP}{239}{subsubsection.11.3.2.1}} \newlabel{openmpmatmul}{{11.3.2.1}{239}{Example: Matrix Multiply in OpenMP}{subsubsection.11.3.2.1}{}} \@writefile{toc}{\contentsline {subsubsection}{\numberline {11.3.2.2}Example: Matrix Multiply in CUDA}{240}{subsubsection.11.3.2.2}} \newlabel{cudamatmult}{{11.3.2.2}{240}{Example: Matrix Multiply in CUDA}{subsubsection.11.3.2.2}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {11.3.3}R Snow}{243}{subsection.11.3.3}} \newlabel{snowmult}{{11.3.3}{243}{R Snow}{subsection.11.3.3}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {11.3.4}R Interfaces to GPUs}{243}{subsection.11.3.4}} \@writefile{toc}{\contentsline {section}{\numberline {11.4}Finding Powers of Matrices}{243}{section.11.4}} \newlabel{matpow}{{11.4}{243}{Finding Powers of Matrices}{section.11.4}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {11.4.1}Example: Graph Connectedness}{243}{subsection.11.4.1}} \@writefile{toc}{\contentsline {subsection}{\numberline {11.4.2}Example: Fibonacci Numbers}{245}{subsection.11.4.2}} \newlabel{fibinit}{{11.27}{245}{Example: Fibonacci Numbers}{equation.11.4.27}{}} \newlabel{fib}{{11.28}{245}{Example: Fibonacci Numbers}{equation.11.4.28}{}} \newlabel{recurs}{{11.29}{245}{Example: Fibonacci Numbers}{equation.11.4.29}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {11.4.3}Example: Matrix Inversion}{246}{subsection.11.4.3}} \newlabel{maxc}{{11.34}{246}{Example: Matrix Inversion}{equation.11.4.34}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {11.4.4}Parallel Computation}{247}{subsection.11.4.4}} \@writefile{toc}{\contentsline {section}{\numberline {11.5}Solving Systems of Linear Equations}{247}{section.11.5}} \newlabel{solvelin}{{11.5}{247}{Solving Systems of Linear Equations}{section.11.5}{}} \newlabel{linsyst}{{11.36}{247}{Solving Systems of Linear Equations}{equation.11.5.36}{}} \newlabel{lin}{{11.37}{247}{Solving Systems of Linear Equations}{equation.11.5.37}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {11.5.1}Gaussian Elimination}{247}{subsection.11.5.1}} \newlabel{gauss}{{11.5.1}{247}{Gaussian Elimination}{subsection.11.5.1}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {11.5.2}Example: Gaussian Elimination in CUDA}{248}{subsection.11.5.2}} \@writefile{toc}{\contentsline {subsection}{\numberline {11.5.3}The Jacobi Algorithm}{249}{subsection.11.5.3}} \newlabel{jacob}{{11.38}{249}{The Jacobi Algorithm}{equation.11.5.38}{}} \newlabel{jacmat}{{11.39}{250}{The Jacobi Algorithm}{equation.11.5.39}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {11.5.4}Example: OpenMP Implementation of the Jacobi Algorithm}{250}{subsection.11.5.4}} \newlabel{ompjacobi}{{11.5.4}{250}{Example: OpenMP Implementation of the Jacobi Algorithm}{subsection.11.5.4}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {11.5.5}Example: R/gputools Implementation of Jacobi}{251}{subsection.11.5.5}} \@writefile{toc}{\contentsline {section}{\numberline {11.6}Eigenvalues and Eigenvectors}{252}{section.11.6}} \newlabel{eigen}{{11.6}{252}{Eigenvalues and Eigenvectors}{section.11.6}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {11.6.1}The Power Method}{252}{subsection.11.6.1}} \newlabel{xkeqn}{{11.40}{252}{The Power Method}{equation.11.6.40}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {11.6.2}Parallel Computation}{253}{subsection.11.6.2}} \@writefile{toc}{\contentsline {section}{\numberline {11.7}Sparse Matrices}{253}{section.11.7}} \newlabel{sparse}{{11.7}{253}{Sparse Matrices}{section.11.7}{}} \newlabel{amat}{{11.48}{254}{Sparse Matrices}{equation.11.7.48}{}} \@writefile{toc}{\contentsline {section}{\numberline {11.8}Libraries}{255}{section.11.8}} \@writefile{toc}{\contentsline {chapter}{\numberline {12}Introduction to Parallel Sorting}{257}{chapter.12}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \newlabel{chap:sort}{{12}{257}{Introduction to Parallel Sorting}{chapter.12}{}} \@writefile{toc}{\contentsline {section}{\numberline {12.1}Quicksort}{257}{section.12.1}} \@writefile{toc}{\contentsline {subsection}{\numberline {12.1.1}The Separation Process}{257}{subsection.12.1.1}} \@writefile{toc}{\contentsline {subsection}{\numberline {12.1.2}Example: OpenMP Quicksort}{259}{subsection.12.1.2}} \newlabel{sharedqs}{{12.1.2}{259}{Example: OpenMP Quicksort}{subsection.12.1.2}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {12.1.3}Hyperquicksort}{260}{subsection.12.1.3}} \@writefile{toc}{\contentsline {section}{\numberline {12.2}Mergesorts}{261}{section.12.2}} \newlabel{mergesort}{{12.2}{261}{Mergesorts}{section.12.2}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {12.2.1}Sequential Form}{261}{subsection.12.2.1}} \@writefile{toc}{\contentsline {subsection}{\numberline {12.2.2}Shared-Memory Mergesort}{261}{subsection.12.2.2}} \@writefile{toc}{\contentsline {subsection}{\numberline {12.2.3}Message Passing Mergesort on a Tree Topology}{261}{subsection.12.2.3}} \newlabel{basicstuff}{{12.2.3}{261}{Message Passing Mergesort on a Tree Topology}{subsection.12.2.3}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {12.2.4}Compare-Exchange Operations}{262}{subsection.12.2.4}} \newlabel{compareexchange}{{12.2.4}{262}{Compare-Exchange Operations}{subsection.12.2.4}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {12.2.5}Bitonic Mergesort}{262}{subsection.12.2.5}} \@writefile{toc}{\contentsline {section}{\numberline {12.3}The Bubble Sort and Its Cousins}{264}{section.12.3}} \@writefile{toc}{\contentsline {subsection}{\numberline {12.3.1}The Much-Maligned Bubble Sort}{264}{subsection.12.3.1}} \@writefile{toc}{\contentsline {subsection}{\numberline {12.3.2}A Popular Variant: Odd-Even Transposition}{265}{subsection.12.3.2}} \@writefile{toc}{\contentsline {subsection}{\numberline {12.3.3}Example: CUDA Implementation of Odd/Even Transposition Sort}{265}{subsection.12.3.3}} \newlabel{cudaoddeven}{{12.3.3}{265}{Example: CUDA Implementation of Odd/Even Transposition Sort}{subsection.12.3.3}{}} \@writefile{toc}{\contentsline {section}{\numberline {12.4}Shearsort}{267}{section.12.4}} \@writefile{toc}{\contentsline {section}{\numberline {12.5}Bucket Sort with Sampling}{267}{section.12.5}} \newlabel{bsort}{{12.5}{267}{Bucket Sort with Sampling}{section.12.5}{}} \@writefile{toc}{\contentsline {section}{\numberline {12.6}Radix Sort}{271}{section.12.6}} \@writefile{toc}{\contentsline {section}{\numberline {12.7}Enumeration Sort}{271}{section.12.7}} \@writefile{toc}{\contentsline {chapter}{\numberline {13}Parallel Computation for Audio and Image Processing}{273}{chapter.13}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \newlabel{chap:fft}{{13}{273}{Parallel Computation for Audio and Image Processing}{chapter.13}{}} \@writefile{toc}{\contentsline {section}{\numberline {13.1}General Principles}{273}{section.13.1}} \@writefile{toc}{\contentsline {subsection}{\numberline {13.1.1}One-Dimensional Fourier Series}{273}{subsection.13.1.1}} \newlabel{poly}{{13.2}{274}{One-Dimensional Fourier Series}{equation.13.1.2}{}} \newlabel{exptaylor}{{13.4}{274}{One-Dimensional Fourier Series}{equation.13.1.4}{}} \newlabel{f2t}{{13.5}{275}{One-Dimensional Fourier Series}{equation.13.1.5}{}} \newlabel{t2fi}{{13.8}{275}{One-Dimensional Fourier Series}{equation.13.1.8}{}} \newlabel{t2fii}{{13.9}{275}{One-Dimensional Fourier Series}{equation.13.1.9}{}} \newlabel{t2fiii}{{13.10}{275}{One-Dimensional Fourier Series}{equation.13.1.10}{}} \newlabel{compact}{{13.12}{277}{One-Dimensional Fourier Series}{equation.13.1.12}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {13.1.2}Two-Dimensional Fourier Series}{277}{subsection.13.1.2}} \@writefile{toc}{\contentsline {section}{\numberline {13.2}Discrete Fourier Transforms}{277}{section.13.2}} \newlabel{1dim}{{13.2}{277}{Discrete Fourier Transforms}{section.13.2}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {13.2.1}One-Dimensional Data}{278}{subsection.13.2.1}} \newlabel{onedimdft}{{13.2.1}{278}{One-Dimensional Data}{subsection.13.2.1}{}} \newlabel{dft}{{13.13}{278}{One-Dimensional Data}{equation.13.2.13}{}} \newlabel{CfromX}{{13.16}{278}{One-Dimensional Data}{equation.13.2.16}{}} \newlabel{vander}{{13.17}{279}{One-Dimensional Data}{equation.13.2.17}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {13.2.2}Inversion}{279}{subsection.13.2.2}} \newlabel{vanderinv}{{13.18}{279}{Inversion}{equation.13.2.18}{}} \newlabel{XfromC}{{13.19}{279}{Inversion}{equation.13.2.19}{}} \newlabel{dftinv}{{13.20}{280}{Inversion}{equation.13.2.20}{}} \@writefile{toc}{\contentsline {subsubsection}{\numberline {13.2.2.1}Alternate Formulation}{280}{subsubsection.13.2.2.1}} \@writefile{toc}{\contentsline {subsection}{\numberline {13.2.3}Two-Dimensional Data}{280}{subsection.13.2.3}} \newlabel{dft2}{{13.21}{280}{Two-Dimensional Data}{equation.13.2.21}{}} \newlabel{dft2inv}{{13.22}{280}{Two-Dimensional Data}{equation.13.2.22}{}} \@writefile{toc}{\contentsline {section}{\numberline {13.3}Parallel Computation of Discrete Fourier Transforms}{281}{section.13.3}} \@writefile{toc}{\contentsline {subsection}{\numberline {13.3.1}The Fast Fourier Transform}{281}{subsection.13.3.1}} \newlabel{recursive}{{13.24}{281}{The Fast Fourier Transform}{equation.13.3.24}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {13.3.2}A Matrix Approach}{282}{subsection.13.3.2}} \newlabel{matrix}{{13.25}{282}{A Matrix Approach}{equation.13.3.25}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {13.3.3}Parallelizing Computation of the Inverse Transform}{282}{subsection.13.3.3}} \@writefile{toc}{\contentsline {subsection}{\numberline {13.3.4}Parallelizing Computation of the Two-Dimensional Transform}{282}{subsection.13.3.4}} \newlabel{par2d}{{13.3.4}{282}{Parallelizing Computation of the Two-Dimensional Transform}{subsection.13.3.4}{}} \newlabel{dft2new}{{13.26}{282}{Parallelizing Computation of the Two-Dimensional Transform}{equation.13.3.26}{}} \newlabel{zzz}{{13.27}{282}{Parallelizing Computation of the Two-Dimensional Transform}{equation.13.3.26}{}} \@writefile{toc}{\contentsline {section}{\numberline {13.4}Available FFT Software}{283}{section.13.4}} \@writefile{toc}{\contentsline {subsection}{\numberline {13.4.1}R}{283}{subsection.13.4.1}} \newlabel{rfft}{{13.4.1}{283}{R}{subsection.13.4.1}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {13.4.2}CUFFT}{283}{subsection.13.4.2}} \@writefile{toc}{\contentsline {subsection}{\numberline {13.4.3}FFTW}{283}{subsection.13.4.3}} \@writefile{toc}{\contentsline {section}{\numberline {13.5}Applications to Image Processing}{284}{section.13.5}} \@writefile{toc}{\contentsline {subsection}{\numberline {13.5.1}Smoothing}{284}{subsection.13.5.1}} \newlabel{smoothing}{{13.5.1}{284}{Smoothing}{subsection.13.5.1}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {13.5.2}Example: Audio Smoothing in R}{284}{subsection.13.5.2}} \@writefile{toc}{\contentsline {subsection}{\numberline {13.5.3}Edge Detection}{285}{subsection.13.5.3}} \@writefile{toc}{\contentsline {section}{\numberline {13.6}R Access to Sound and Image Files}{286}{section.13.6}} \@writefile{toc}{\contentsline {section}{\numberline {13.7}Keeping the Pixel Intensities in the Proper Range}{286}{section.13.7}} \@writefile{toc}{\contentsline {section}{\numberline {13.8}Does the Function g() Really Have to Be Repeating?}{287}{section.13.8}} \@writefile{toc}{\contentsline {section}{\numberline {13.9}Vector Space Issues (optional section)}{287}{section.13.9}} \newlabel{vector}{{13.9}{287}{Vector Space Issues (optional section)}{section.13.9}{}} \@writefile{toc}{\contentsline {section}{\numberline {13.10}Bandwidth: How to Read the \textit {San Francisco Chronicle} Business Page (optional section)}{289}{section.13.10}} \newlabel{sampling}{{13.10}{289}{Bandwidth: How to Read the \textit {San Francisco Chronicle} Business Page (optional section)}{section.13.10}{}} \@writefile{toc}{\contentsline {chapter}{\numberline {14}Parallel Computation in Statistics/Data Mining}{291}{chapter.14}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \newlabel{chap:stat}{{14}{291}{Parallel Computation in Statistics/Data Mining}{chapter.14}{}} \@writefile{toc}{\contentsline {section}{\numberline {14.1}Itemset Analysis}{291}{section.14.1}} \@writefile{toc}{\contentsline {subsection}{\numberline {14.1.1}What Is It?}{291}{subsection.14.1.1}} \@writefile{toc}{\contentsline {subsection}{\numberline {14.1.2}The Market Basket Problem}{292}{subsection.14.1.2}} \@writefile{toc}{\contentsline {subsection}{\numberline {14.1.3}Serial Algorithms}{293}{subsection.14.1.3}} \@writefile{toc}{\contentsline {subsection}{\numberline {14.1.4}Parallelizing the Apriori Algorithm}{294}{subsection.14.1.4}} \@writefile{toc}{\contentsline {section}{\numberline {14.2}Probability Density Estimation}{294}{section.14.2}} \@writefile{toc}{\contentsline {subsection}{\numberline {14.2.1}Kernel-Based Density Estimation}{295}{subsection.14.2.1}} \newlabel{gaussk}{{14.2}{295}{Kernel-Based Density Estimation}{equation.14.2.2}{}} \newlabel{kernelest}{{14.3}{295}{Kernel-Based Density Estimation}{equation.14.2.3}{}} \newlabel{kernelesta}{{14.5}{297}{}{equation.14.2.5}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {14.2.2}Histogram Computation for Images}{298}{subsection.14.2.2}} \@writefile{toc}{\contentsline {section}{\numberline {14.3}Clustering}{299}{section.14.3}} \newlabel{cluster}{{14.3}{299}{Clustering}{section.14.3}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {14.3.1}Example: k-Means Clustering in R}{301}{subsection.14.3.1}} \@writefile{toc}{\contentsline {section}{\numberline {14.4}Principal Component Analysis (PCA)}{302}{section.14.4}} \newlabel{pca}{{14.4}{302}{Principal Component Analysis (PCA)}{section.14.4}{}} \newlabel{onedim}{{14.7}{302}{Principal Component Analysis (PCA)}{equation.14.4.7}{}} \@writefile{toc}{\contentsline {section}{\numberline {14.5}Monte Carlo Simulation}{303}{section.14.5}} \@writefile{toc}{\contentsline {chapter}{\numberline {A}Miscellaneous Systems Issues}{305}{appendix.A}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \newlabel{chap:os}{{A}{305}{Miscellaneous Systems Issues}{appendix.A}{}} \@writefile{toc}{\contentsline {section}{\numberline {A.1}Timesharing}{305}{section.A.1}} \newlabel{processes}{{A.1}{305}{Timesharing}{section.A.1}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {A.1.1}Many Processes, Taking Turns}{305}{subsection.A.1.1}} \@writefile{toc}{\contentsline {section}{\numberline {A.2}Memory Hierarchies}{307}{section.A.2}} \@writefile{toc}{\contentsline {subsection}{\numberline {A.2.1}Cache Memory}{307}{subsection.A.2.1}} \newlabel{cache}{{A.2.1}{307}{Cache Memory}{subsection.A.2.1}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {A.2.2}Virtual Memory}{307}{subsection.A.2.2}} \newlabel{vm}{{A.2.2}{307}{Virtual Memory}{subsection.A.2.2}{}} \newlabel{goals}{{A.2.2.1}{307}{Make Sure You Understand the Goals}{subsubsection.A.2.2.1}{}} \@writefile{toc}{\contentsline {subsubsection}{\numberline {A.2.2.1}Make Sure You Understand the Goals}{307}{subsubsection.A.2.2.1}} \@writefile{toc}{\contentsline {subsubsection}{\numberline {A.2.2.2}How It Works}{308}{subsubsection.A.2.2.2}} \newlabel{howvmworks}{{A.2.2.2}{308}{How It Works}{subsubsection.A.2.2.2}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {A.2.3}Performance Issues}{309}{subsection.A.2.3}} \@writefile{toc}{\contentsline {section}{\numberline {A.3}Array Issues}{309}{section.A.3}} \@writefile{toc}{\contentsline {subsection}{\numberline {A.3.1}Storage}{309}{subsection.A.3.1}} \newlabel{arraystorage}{{A.3.1}{309}{Storage}{subsection.A.3.1}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {A.3.2}Subarrays}{310}{subsection.A.3.2}} \@writefile{toc}{\contentsline {subsection}{\numberline {A.3.3}Memory Allocation}{310}{subsection.A.3.3}} \@writefile{toc}{\contentsline {chapter}{\numberline {B}Review of Matrix Algebra}{313}{appendix.B}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \newlabel{chap:matrixreview}{{B}{313}{Review of Matrix Algebra}{appendix.B}{}} \@writefile{toc}{\contentsline {section}{\numberline {B.1}Terminology and Notation}{313}{section.B.1}} \@writefile{toc}{\contentsline {subsection}{\numberline {B.1.1}Matrix Addition and Multiplication}{314}{subsection.B.1.1}} \@writefile{toc}{\contentsline {section}{\numberline {B.2}Matrix Transpose}{315}{section.B.2}} \@writefile{toc}{\contentsline {section}{\numberline {B.3}Linear Independence}{316}{section.B.3}} \@writefile{toc}{\contentsline {section}{\numberline {B.4}Determinants}{316}{section.B.4}} \@writefile{toc}{\contentsline {section}{\numberline {B.5}Matrix Inverse}{316}{section.B.5}} \newlabel{matrixinverse}{{B.5}{316}{Matrix Inverse}{section.B.5}{}} \newlabel{qr}{{B.18}{317}{Matrix Inverse}{equation.B.5.18}{}} \@writefile{toc}{\contentsline {section}{\numberline {B.6}Eigenvalues and Eigenvectors}{317}{section.B.6}} \newlabel{diagonalize}{{B.20}{317}{Eigenvalues and Eigenvectors}{equation.B.6.20}{}} \@writefile{toc}{\contentsline {section}{\numberline {B.7}Rank of a Matrix}{318}{section.B.7}} \@writefile{toc}{\contentsline {section}{\numberline {B.8}Matrix Algebra in R}{318}{section.B.8}} \@writefile{toc}{\contentsline {chapter}{\numberline {C}R Quick Start}{321}{appendix.C}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \newlabel{chap:rquickstart}{{C}{321}{R Quick Start}{appendix.C}{}} \@writefile{toc}{\contentsline {section}{\numberline {C.1}Correspondences}{321}{section.C.1}} \@writefile{toc}{\contentsline {section}{\numberline {C.2}Starting R}{322}{section.C.2}} \@writefile{toc}{\contentsline {section}{\numberline {C.3}First Sample Programming Session}{322}{section.C.3}} \@writefile{toc}{\contentsline {section}{\numberline {C.4}Vectorization}{324}{section.C.4}} \@writefile{toc}{\contentsline {section}{\numberline {C.5}Second Sample Programming Session}{325}{section.C.5}} \@writefile{toc}{\contentsline {section}{\numberline {C.6}Recycling}{326}{section.C.6}} \@writefile{toc}{\contentsline {section}{\numberline {C.7}More on Vectorization}{326}{section.C.7}} \@writefile{toc}{\contentsline {section}{\numberline {C.8}Third Sample Programming Session}{327}{section.C.8}} \@writefile{toc}{\contentsline {section}{\numberline {C.9}Default Argument Values}{328}{section.C.9}} \@writefile{toc}{\contentsline {section}{\numberline {C.10}The R List Type}{328}{section.C.10}} \@writefile{toc}{\contentsline {subsection}{\numberline {C.10.1}The Basics}{328}{subsection.C.10.1}} \@writefile{toc}{\contentsline {subsection}{\numberline {C.10.2}The Reduce() Function}{329}{subsection.C.10.2}} \@writefile{toc}{\contentsline {subsection}{\numberline {C.10.3}S3 Classes}{330}{subsection.C.10.3}} \@writefile{toc}{\contentsline {section}{\numberline {C.11}Some Workhorse Functions}{331}{section.C.11}} \@writefile{toc}{\contentsline {section}{\numberline {C.12}Handy Utilities}{332}{section.C.12}} \@writefile{toc}{\contentsline {section}{\numberline {C.13}Data Frames}{334}{section.C.13}} \@writefile{toc}{\contentsline {section}{\numberline {C.14}Graphics}{335}{section.C.14}} \@writefile{toc}{\contentsline {section}{\numberline {C.15}Packages}{336}{section.C.15}} \@writefile{toc}{\contentsline {section}{\numberline {C.16}Other Sources for Learning R}{336}{section.C.16}} \@writefile{toc}{\contentsline {section}{\numberline {C.17}Online Help}{337}{section.C.17}} \@writefile{toc}{\contentsline {section}{\numberline {C.18}Debugging in R}{337}{section.C.18}} \@writefile{toc}{\contentsline {section}{\numberline {C.19}Complex Numbers}{337}{section.C.19}} \@writefile{toc}{\contentsline {section}{\numberline {C.20}Further Reading}{338}{section.C.20}}