.NET实现 数据提取、转换和加载

早晨看到一篇文章：

http://www.codeproject.com/Articles/34556/Write-ETL-jobs-in-pure-C

 

有二个文件：User name file(ID NAME)、Addresses file(ID ADDRESS):

Id Name

1  Bob

Id Address

1 123 Main St.

2 42 Everywhich way
如果想得到

Id Name Address 如果在SQL中一个查询 就可得到。 在文件处理的过程中，基于c# 如何处理呢？ 代码中给出了解决方案。

首先把二个文件的内容读取：

FileEngine是基于 FileHelpers 

public class UserNameRead : AbstractOperation

{

    public UserNameRead(string filePath)

    {

        this.filePath = filePath;

    }



    string filePath = null;



    public override IEnumerable<Row> Execute(IEnumerable<Row> rows)

    {

        using (FileEngine file = FluentFile.For<UserNameRecord>().From(filePath))

        {

            foreach (object obj in file)

            {

                yield return Row.FromObject(obj);

            }

        }

    }

}



public class UserAddressRead : AbstractOperation

{

    public UserAddressRead(string filePath)

    {

        this.filePath = filePath;

    }



    string filePath = null;



    public override IEnumerable<Row> Execute(IEnumerable<Row> rows)

    {

        using (FileEngine file = FluentFile.For<UserAddressRecord>().From(filePath))

        {

            foreach (object obj in file)

            {

                yield return Row.FromObject(obj);

            }

        }

    }

}

创建二个文件的关系并构造新的文件

public class JoinUserRecords : JoinOperation

{

    protected override void SetupJoinConditions()

    {

        InnerJoin

            .Left("Id")

            .Right("Id");

    }



    protected override Row MergeRows(Row leftRow, Row rightRow)

    {

        Row row = new Row();

        row.Copy(leftRow);



        //copy over all properties not in the user records

        row["Address"] = rightRow["Address"];



        return row;

    }

}


创建好的结构 如何输出：

public class UserFullWrite : AbstractOperation

{

    public UserFullWrite(string filePath)

    {

        this.filePath = filePath;

    }



    string filePath = null;



    public override IEnumerable<Row> Execute(IEnumerable<Row> rows)

    {

        FluentFile engine = FluentFile.For<UserFullRecord>();

        engine.HeaderText = "Id\tName\tAddress";

        using (FileEngine file = engine.To(filePath))

        {

            foreach (Row row in rows)

            {

                file.Write(row.ToObject<UserFullRecord>());



                //pass through rows if needed for another later operation 

                yield return row;

            }

        }

    }

}

调用方法：

public class MainProcess : EtlProcess

{

    protected override void Initialize()

    {

        Register(new JoinUserRecords()

            .Left(new UserNameRead(Settings.Default.NamesFile))

            .Right(new UserAddressRead(Settings.Default.AddressesFile))

        );



        Register(new UserFullWrite(Settings.Default.OutputFile));

    }

}

总结： 对于结构化的文件 ， 通常比较好处理，但是对于非结构化的文件处理，不易处理。