parallel, pipelined, buffer, unload data

1) parallel, pipelined, buffer, unload data
1@@@@ Asynchronous Data Unloading with Parallel Pipelined Functions   
referencing "PLSQL Design 5th Edition"
So far, I have demonstrated two types of data loads that have benefited from conversion
to a parallel pipelined function. You might also want to exploit the parallel feature of
pipelined functions for those times when you need to unload data (even well into the
21st century I have yet to see a corporate in-house ODS/DSS/warehouse that doesn¡¯t
extract data for transfer to other systems).

Prepare:
   @@@Create a directory object
   CREATE DIRECTORY dir01 AS '/u01/app/oracle/dir01';
   GRANT ALL ON DIRECTORY TO hr;

   @@@Create a 1 million table
   CREATE TABLE source_table AS SELECT * FROM dba_source;
   INSERT INTO source_table SELECT * FROM source_table;
   INSERT INTO source_table SELECT * FROM source_table;
   COMMIT;

Example:
   @@@Create Three methods to unload data from oracle 10g db.
    HR@ocm> !cat tmp.sql
    DROP TYPE display_table_ot
    /
    CREATE TYPE display_table_ot AS OBJECT
        (  file_name   VARCHAR2(128)
         , no_records  NUMBER
         , session_id  NUMBER   );
    /
    CREATE OR REPLACE PACKAGE unload_pkg
    IS
    /*
      FUNCTION  parallel_unload
      FUNCTION  parallel_unload_buffered
      PROCEDURE legacy_unload
   
    --test
      PROCEDURE test_legacy_unload
      PROCEDURE test_pl_unload
      PROCEDURE test_pl_unload_buffered
    */
   
      c_limit CONSTANT PLS_INTEGER := 100;
   
      TYPE display_table_ntt IS TABLE OF display_table_ot;
   
      FUNCTION parallel_unload
            (  p_source_in       IN  SYS_REFCURSOR
             , p_filename_in     IN  VARCHAR2
             , p_directory_in    IN  VARCHAR2
             , p_limit_size_in   IN  PLS_INTEGER DEFAULT unload_pkg.c_limit   )
        RETURN display_table_ntt
        PIPELINED
        PARALLEL_ENABLE ( PARTITION p_source_in BY ANY);
   
      FUNCTION parallel_unload_buffered
            (  p_source_in        IN  SYS_REFCURSOR
             , p_filename_in      IN  VARCHAR2
             , p_directory_in     IN  VARCHAR2
             , p_limit_size_in    IN  PLS_INTEGER DEFAULT unload_pkg.c_limit  )
        RETURN display_table_ntt
        PIPELINED
        PARALLEL_ENABLE ( PARTITION p_source_in BY ANY);
   
      PROCEDURE legacy_unload
            (  p_source_in       IN  SYS_REFCURSOR  --weak type
             , p_filename_in     IN  VARCHAR2
             , p_directory_in    IN  VARCHAR2
             , p_limit_size_in   IN  PLS_INTEGER DEFAULT unload_pkg.c_limit  );
   
      PROCEDURE test_legacy_unload;
      PROCEDURE test_pl_unload;
      PROCEDURE test_pl_unload_buffered;
   
    END unload_pkg;
    /
    HR@ocm> @tmp.sql
    Type dropped.
    Type created.
    Package created.
   
   
   @@@Create package body, here are three method and three procedure to test them.
    HR@ocm> !cat tmpx.sql
    CREATE OR REPLACE PACKAGE BODY unload_pkg
    IS
    /*
    Overview:
      Part 1#
      1.1 (Procedure) legacy_unload
      1.2 (Function ) parallel_unload
      1.3 (Function ) parallel_unload_buffered 
   
      Part 2# test
      2.1 (Procedrue) test_legacy_unload;
      2.2 (Procedure) test_pl_unload;
      2.3 (Procedure) test_pl_unload_buffered;
    */    
   
        gc_maxline  CONSTANT PLS_INTEGER := 32767;
   
        TYPE g_row_aat IS TABLE OF VARCHAR2(32767)
           INDEX BY PLS_INTEGER;
   
    /*
      Part 1#
      1.1 (Procedure) legacy_unload
      1.2 (Function ) parallel_unload
      1.3 (Function ) parallel_unload_buffered 
    */    
    ------------------------------------------------------------------------------
    --1.1)  (Procedure) legacy_unload
   
        PROCEDURE legacy_unload
            (  p_source_in       IN  SYS_REFCURSOR
             , p_filename_in     IN  VARCHAR2
             , p_directory_in    IN  VARCHAR2
             , p_limit_size_in   IN  PLS_INTEGER DEFAULT unload_pkg.c_limit )
        IS
           row_aa   g_row_aat;
           lv_name VARCHAR2(128) := p_filename_in || '.txt';
           lv_file  UTL_FILE.file_type;
        BEGIN
   
           lv_file  := UTL_FILE.fopen( p_directory_in, lv_name, 'w', gc_maxline);
   
           LOOP
           FETCH p_source_in BULK COLLECT INTO row_aa LIMIT p_limit_size_in;
           EXIT WHEN row_aa.COUNT = 0; --traversal all rows
              FOR i IN 1 .. row_aa.COUNT
              LOOP
                 UTL_FILE.put_line(lv_file, row_aa(i));
              END LOOP;
           END LOOP;      
           DBMS_OUTPUT.put_line(p_source_in%ROWCOUNT||' rows unloaded.');
           CLOSE p_source_in;
   
           UTL_FILE.fclose(lv_file);
   
        END legacy_unload;
   
   
    --1.2) (Function) parallel_unload
   
        FUNCTION parallel_unload
            (  p_source_in      IN   SYS_REFCURSOR
             , p_filename_in    IN   VARCHAR2
             , p_directory_in   IN   VARCHAR2
             , p_limit_size_in  IN   PLS_INTEGER DEFAULT unload_pkg.c_limit )
          RETURN display_table_ntt
          PIPELINED
          PARALLEL_ENABLE (PARTITION p_source_in BY ANY)
        IS
           rows_aa   g_row_aat;
           lv_sid    NUMBER := SYS_CONTEXT('USERENV','SID');
           lv_fname  VARCHAR2(128) := p_filename_in ||'.txt';
           lv_file   UTL_FILE.file_type;
           lv_lines  PLS_INTEGER;
        BEGIN
          
           lv_file := UTL_FILE.fopen(p_directory_in, lv_fname, 'w', gc_maxline );      
   
           LOOP
           FETCH p_source_in BULK COLLECT INTO rows_aa LIMIT p_limit_size_in;
           EXIT WHEN rows_aa.COUNT = 0 ; --all
              FOR i IN  1 .. rows_aa.COUNT
              LOOP
                 UTL_FILE.put_line(lv_file, rows_aa(i));
              END LOOP;
           END LOOP;
   
           lv_lines := p_source_in%ROWCOUNT;
           CLOSE p_source_in; 
           UTL_FILE.fclose(lv_file); --not the filename, is the file type;
       
           PIPE ROW ( display_table_ot(lv_fname,lv_lines,lv_sid) );
           RETURN;    
        END parallel_unload;
   
   
    --1.3 (Function) parallel_unload_buffered
    --    Write content of a cursor to a file within a directory. buffer cache 32767 char|byte
    --    Then wirte the buffer into that file and reinitial buffer cache
       
        FUNCTION parallel_unload_buffered
            (  p_source_in      IN  SYS_REFCURSOR
             , p_filename_in    IN  VARCHAR2
             , p_directory_in   IN  VARCHAR2
             , p_limit_size_in  IN  PLS_INTEGER DEFAULT unload_pkg.c_limit )
          RETURN display_table_ntt
          PIPELINED
          PARALLEL_ENABLE (PARTITION p_source_in BY ANY)
        IS
           c_eol      CONSTANT VARCHAR2(1) := chr(10);
           rows_aa    g_row_aat;
           lv_buffer  VARCHAR2(32767);
           lv_sid     NUMBER := SYS_CONTEXT('USERENV','SID');
           lv_fname   VARCHAR2(128) := p_filename_in||'_'||lv_sid||'.txt';
           lv_file    UTL_FILE.file_type;
           lv_lines   PLS_INTEGER;
        BEGIN   
           lv_file  := UTL_FILE.fopen( p_directory_in, lv_fname, 'w', gc_maxline );
          
           LOOP
              FETCH p_source_in BULK COLLECT INTO rows_aa LIMIT p_limit_size_in;   
              EXIT WHEN rows_aa.COUNT=0 ;
              FOR i IN  1 .. rows_aa.COUNT
              LOOP
                 IF length(lv_buffer) + length(rows_aa(i)) <= gc_maxline - 2000 THEN
                    lv_buffer := lv_buffer || c_eol || rows_aa(i) ;
                 ELSE
                    IF lv_buffer IS NOT NULL THEN
                       UTL_FILE.put_line( lv_file, lv_buffer );
                    END IF;
                 END IF;                      
              END LOOP;
           END LOOP;
           UTL_FILE.put_line( lv_file, lv_buffer ); --wirte the rest in buffer.
   
           lv_lines := p_source_in%ROWCOUNT;
   
           CLOSE p_source_in;
           UTL_FILE.fclose(lv_file);
   
           PIPE ROW ( display_table_ot(lv_fname, lv_lines, lv_sid) );
           RETURN;
   
        END parallel_unload_buffered;   
   
   
    /*
      Part 2# test
      2.1 (Procedrue) test_legacy_unload;
      2.2 (Procedure) test_pl_unload;
      2.3 (Procedure) test_pl_unload_buffered;
    */    
    ---------------------------------------------------------------------------------------
    --2.1) (Procedrue) test_legacy_unload
   
        PROCEDURE test_legacy_unload
        IS
           lv_cur  SYS_REFCURSOR;
        BEGIN
           OPEN  lv_cur FOR
              SELECT owner||','||name||','||type||','||
                     line||','||substr(text,1,30)  AS dba_source
                FROM source_table;
   
           unload_pkg.legacy_unload
              (  lv_cur                  --cursor
               , 'legacy_unload_output'  --filename
               , 'DIR01'     );          --dir
   
        END test_legacy_unload;
   
   
    --2.2) (Procedure) test_pl_unload
   
        PROCEDURE test_pl_unload
        IS
           CURSOR cur IS
            SELECT *
              FROM TABLE(
                       unload_pkg.parallel_unload (
                          CURSOR( SELECT /*+ PARALLEL(s,4) */
                                         owner||','||name||','||type||','||
                                         line ||','||substr(text,1,30)
                                    FROM source_table s )   --cursor
                        , 'pl_unload_output'                --filename        
                        , 'DIR01'  ) );                     --dir
            lv_filename   VARCHAR2(128);
            lv_rec        NUMBER;
            lv_sid        NUMBER;
   
        BEGIN
           DBMS_OUTPUT.put_line('Test parallel pipelined unload:');
           DBMS_OUTPUT.put_line('Filename                   Records      Session ID');
           OPEN  cur;
           LOOP
           FETCH cur INTO lv_filename, lv_rec, lv_sid;
           EXIT WHEN cur%NOTFOUND;
              DBMS_OUTPUT.put_line(lv_filename||' , '||lv_rec||' , '||lv_sid);
           END LOOP;
           CLOSE cur;
        END test_pl_unload;
   
   
    --2.3) (Procedure) test_pl_unload_buffered
   
        PROCEDURE test_pl_unload_buffered
        IS
           CURSOR cur IS
            SELECT *
              FROM TABLE(
                       unload_pkg.parallel_unload_buffered (
                          CURSOR( SELECT /*+ PARALLEL(s,4) */
                                         owner||','||name||','||type||','||
                                         line ||','||substr(text,1,30)
                                    FROM source_table s )   --cursor
                        , 'pl_unload_output_buffered'       --filename        
                        , 'DIR01'  ) );                     --dir
            lv_filename   VARCHAR2(128);
            lv_rec        NUMBER;
            lv_sid        NUMBER;
        BEGIN
           DBMS_OUTPUT.put_line('Test parallel pipelined unload:');
           DBMS_OUTPUT.put_line('Filename                   Records      Session ID');
           OPEN  cur;
           LOOP
           FETCH cur INTO lv_filename, lv_rec, lv_sid ;
           EXIT WHEN cur%NOTFOUND;
              DBMS_OUTPUT.put_line(lv_filename||' , '||lv_rec||' , '||lv_sid);
           END LOOP;
           CLOSE cur;
        END test_pl_unload_buffered;
   
    END unload_pkg;
    /
    HR@ocm> @tmpx.sql
    Package body created.
   
   
   
   @@@Begin to test.   
    HR@ocm> set serveroutput on;
    HR@ocm> set timing on;
   
    HR@ocm> exec unload_pkg.test_legacy_unload;
    1180796 rows unloaded.
    PL/SQL procedure successfully completed.
    Elapsed: 00:00:09.25
   
    HR@ocm> exec unload_pkg.test_pl_unload;
    PL/SQL procedure successfully completed.
    Elapsed: 00:00:05.19
   
    HR@ocm> exec unload_pkg.test_pl_unload_buffered;
    PL/SQL procedure successfully completed.
    Elapsed: 00:00:00.92
   
   @@@Summary:
    buffer + pipelined function + parallel would work best. It is the stream function.
    Very very good performance to unload 1 millions.