Caiso

README.md

@@ -7,8 +7,8 @@
 
 |     Data      |  PJM  | MISO  | CAISO |
 | ------------- | :---: | :---: | :---: |
-| Real Time LMP |  ❌  |  ✔️   |   ❌   |
-| Day-ahead LMP |  ✔️  |  ✔️   |   ❌   |
+| Real Time LMP |  ❌  |  ✔️   |   ✔️   |
+| Day-ahead LMP |  ✔️  |  ✔️   |   ✔️   |
 | Load          |  ❌  |   ❌   |   ❌   |
 
 Example of getting data from PJM

src/ElectricityMarketData.jl

@@ -25,6 +25,7 @@ export ElectricityMarket,
 
 # general
 include("helpers/http_helper.jl")
+include("helpers/zip_helper.jl")
 include("electricity_market.jl")
 
 #miso
@@ -37,4 +38,10 @@ include("pjm/urls.jl")
 include("pjm/utils.jl")
 include("pjm/parser.jl")
 
+#caiso
+include("caiso/caiso_market.jl")
+include("caiso/datetime.jl")
+include("caiso/urls.jl")
+include("caiso/utils.jl")
+
 end # module

src/caiso/caiso_market.jl

@@ -0,0 +1,144 @@
+"""
+    CaisoMarket
+
+struct representing the Midcontinent Independent System Operator (MISO).
+"""
+
+Base.@kwdef struct CaisoMarket <: ElectricityMarket
+    url::String = "https://www.caiso.com/"
+    directory::String = "CaisoMarket"
+    timezone::TimeZone = tz"UTC-8"
+end
+
+"""
+    available_time_series(::CaisoMarket) :: Vector{NamedTuple}
+
+Return Vector of `NamedTuple` of available time series for the given `market`.
+
+Ex:
+```
+[
+    (name="RT-load", unit="MW", resolution=Hour(1), first_date=DateTime("2021-01-01T00:00:00"), method=get_real_time_load_data, description="Real-time load data"),
+    (name="RT-LMP", unit="MWh", resolution=Hour(1), first_date=DateTime("2021-01-01T00:00:00"), method=get_real_time_lmp, description="Real-time Locational Marginal Price data"),
+]
+```
+"""
+function available_time_series(::CaisoMarket)::Vector{NamedTuple}
+    return [
+        (
+            name = "RT-LMP",
+            unit = "\$/MWh",
+            resolution = Hour(1),
+            first_date = DateTime("2005-05-01T00:00:00"),
+            method = get_real_time_lmp,
+            description = "Real-time Locational Marginal Price data",
+        ),
+        (
+            name = "DA-LMP",
+            unit = "\$/MWh",
+            resolution = Hour(1),
+            first_date = DateTime("2005-05-01T00:00:00"),
+            method = get_day_ahead_lmp,
+            description = "Day-ahead Locational Marginal Price data",
+        ),
+    ]
+end
+
+"""
+    get_timezone(market::CaisoMarket) :: TimeZone
+
+Return the timezone of the given `market`.
+"""
+function get_timezone(market::CaisoMarket)::TimeZone
+    return market.timezone
+end
+
+"""
+    get_real_time_lmp_raw_data(market::CaisoMarket, start_date::ZonedDateTime, end_date::ZonedDateTime; folder::AbstractString=tempdir())
+
+Download raw data for Real-Time (RT) Locational Marginal Price (LMP) for the given `market` and `start_date` to `end_date` and save it in `folder`.
+"""
+function get_real_time_lmp_raw_data(
+    market::CaisoMarket,
+    start_date::ZonedDateTime,
+    end_date::ZonedDateTime;
+    folder::AbstractString = tempdir(),
+)::Nothing
+    return _get_raw_data(
+        market,
+        "RTM",
+        Hour(1),
+        _get_caiso_real_time_url,
+        start_date,
+        end_date,
+        folder,
+    )
+end
+
+"""
+    get_day_ahead_lmp_raw_data(market::CaisoMarket, start_date::ZonedDateTime, end_date::ZonedDateTime; folder::AbstractString=tempdir())
+
+Download raw data for Day-Ahead (DA) Locational Marginal Price (LMP) for the given `market` and `start_date` to `end_date` and save it in `folder`.
+"""
+function get_day_ahead_lmp_raw_data(
+    market::CaisoMarket,
+    start_date::ZonedDateTime,
+    end_date::ZonedDateTime;
+    folder::AbstractString = tempdir(),
+)::Nothing
+    return _get_raw_data(
+        market,
+        "DAM",
+        Day(1),
+        _get_caiso_day_ahead_url,
+        start_date,
+        end_date,
+        folder,
+    )
+end
+
+"""
+    get_real_time_lmp(market::CaisoMarket, start_date::ZonedDateTime, end_date::ZonedDateTime; folder::AbstractString=tempdir()) :: Tables.table
+
+Return a table with Real-Time (RT) Locational Marginal Price (LMP) data for the given `market` and `start_date` to `end_date`.
+If the data is not available, download it and save it in `folder`. 
+"""
+function get_real_time_lmp(
+    market::CaisoMarket,
+    start_date::ZonedDateTime,
+    end_date::ZonedDateTime;
+    folder::AbstractString = tempdir(),
+)
+    return _get_data(
+        market,
+        "RTM",
+        Hour(1),
+        _get_caiso_real_time_url,
+        start_date,
+        end_date,
+        folder,
+    )
+end
+
+"""
+    get_day_ahead_lmp(market::CaisoMarket, start_date::ZonedDateTime, end_date::ZonedDateTime; folder::AbstractString=tempdir()) :: Tables.table
+
+Return a table with Day-Ahead (DA) Locational Marginal Price (LMP) data for the given `market` and `start_date` to `end_date`.
+If the data is not available, download it and save it in `folder`. 
+"""
+function get_day_ahead_lmp(
+    market::CaisoMarket,
+    start_date::ZonedDateTime,
+    end_date::ZonedDateTime;
+    folder::AbstractString = tempdir(),
+)
+    return _get_data(
+        market,
+        "DAM",
+        Day(1),
+        _get_caiso_day_ahead_url,
+        start_date,
+        end_date,
+        folder,
+    )
+end

src/caiso/datetime.jl

@@ -0,0 +1,11 @@
+"""
+    _get_zdt_formated(zdt::ZonedDateTime)::AbstractString
+
+Get the zdt formated string in the format "yyyymmddTHH:MMzzzz".
+"""
+function _get_zdt_formated(zdt::ZonedDateTime)::AbstractString
+    return string(
+        Dates.format(zdt, "yyyymmddTHH:MM"),
+        replace(Dates.format(zdt, "zzzz"), ":" => ""),
+    )
+end

src/caiso/urls.jl

@@ -0,0 +1,39 @@
+"""
+    _get_caiso_url(queryname::AbstractString, startdate::AbstractString, stopdate::AbstractString, market_run_id::AbstractString, version::AbstractString = "1", resultformat::AbstractString="6")::AbstractString
+
+Returns the url for the Caiso API.
+"""
+function _get_caiso_url(
+    queryname::AbstractString,
+    startdate::AbstractString,
+    stopdate::AbstractString,
+    market_run_id::AbstractString,
+    version::AbstractString = "1",
+    resultformat::AbstractString = "6",
+)::AbstractString
+    return "http://oasis.caiso.com/oasisapi/SingleZip?resultformat=$resultformat&queryname=$queryname&version=$version&startdatetime=$startdate&enddatetime=$stopdate&market_run_id=$market_run_id"
+end
+
+"""
+    _get_caiso_day_ahead_url(startdate::AbstractString, stopdate::AbstractString)::AbstractString
+
+Returns the day ahead url for the Caiso API.
+"""
+function _get_caiso_day_ahead_url(
+    startdate::AbstractString,
+    stopdate::AbstractString,
+)::AbstractString
+    return _get_caiso_url("PRC_LMP", startdate, stopdate, "DAM")
+end
+
+"""
+    _get_caiso_real_time_url(startdate::AbstractString, stopdate::AbstractString)::AbstractString
+
+Returns the real time url for the Caiso API.
+"""
+function _get_caiso_real_time_url(
+    startdate::AbstractString,
+    stopdate::AbstractString,
+)::AbstractString
+    return _get_caiso_url("PRC_INTVL_LMP", startdate, stopdate, "RTM")
+end

src/caiso/utils.jl

@@ -0,0 +1,100 @@
+"""
+    _async_get_raw_data(market::CaisoMarket, serie::AbstractString, period::Period, func::Function, start_date::ZonedDateTime, end_date::ZonedDateTime; folder::AbstractString)::Vector{Task}
+
+Download the 'func' raw data for the given `market` and `start_date` to `end_date` and save it in `folder`.
+"""
+function _async_get_raw_data(
+    market::CaisoMarket,
+    serie::AbstractString,
+    period::Period,
+    func::Function,
+    start_date::ZonedDateTime,
+    end_date::ZonedDateTime,
+    folder::AbstractString,
+)::Vector{Task}
+    directory = mkpath(joinpath(folder, market.directory))
+    tasks = Vector{Task}()
+    date = start_date
+    while date < end_date
+        start_formated = _get_zdt_formated(date)
+        next = date + period
+        end_formated = _get_zdt_formated(next)
+        url = func(start_formated, end_formated)
+        push!(
+            tasks,
+            _async_download(
+                url,
+                joinpath(
+                    directory,
+                    replace(start_formated, ":" => "") * "_" * serie * ".zip",
+                ),
+            ),
+        )
+        date = next
+    end
+    return tasks
+end
+
+"""
+    _get_raw_data(market::CaisoMarket, serie::AbstractString, period::Period, func::Function, start_date::ZonedDateTime, end_date::ZonedDateTime; folder::AbstractString)::Nothing
+
+Download the 'func' raw data for the given `market` and `start_date` to `end_date` and save it in `folder`.
+"""
+function _get_raw_data(
+    market::CaisoMarket,
+    serie::AbstractString,
+    period::Period,
+    func::Function,
+    start_date::ZonedDateTime,
+    end_date::ZonedDateTime,
+    folder::AbstractString,
+)::Nothing
+    tasks = _async_get_raw_data(market, serie, period, func, start_date, end_date, folder)
+    for task in tasks
+        _ = _unzip(fetch(task), x -> serie * "_" * x)
+    end
+    return nothing
+end
+
+"""
+    _get_data(market::CaisoMarket, serie::AbstractString, period::Period, func::Function, start_date::ZonedDateTime, end_date::ZonedDateTime; folder::AbstractString)
+
+Download the 'func' raw data for the given `market` and `start_date` to `end_date` and save it in `folder`.
+"""
+function _get_data(
+    market::CaisoMarket,
+    serie::AbstractString,
+    period::Period,
+    func::Function,
+    start_date::ZonedDateTime,
+    end_date::ZonedDateTime,
+    folder::AbstractString,
+)
+    tasks = _async_get_raw_data(market, serie, period, func, start_date, end_date, folder)
+    dfs = Vector{DataFrame}()
+    for task in tasks
+        temp = nothing
+        for file_name in _unzip(fetch(task), x -> serie * "_" * x)
+            df = CSV.read(file_name, DataFrame)
+            pivot =
+                unstack(df, [:INTERVALSTARTTIME_GMT, :MARKET_RUN_ID, :NODE], :LMP_TYPE, :MW)
+            temp =
+                isnothing(temp) ? pivot :
+                outerjoin(temp, pivot, on = [:INTERVALSTARTTIME_GMT, :MARKET_RUN_ID, :NODE])
+        end
+        push!(dfs, temp)
+    end
+    df = vcat(dfs...)
+    rename!(
+        df,
+        Dict(
+            :INTERVALSTARTTIME_GMT => :Time,
+            :MARKET_RUN_ID => :Market,
+            :NODE => :Node,
+            :MCC => :Congestion,
+            :MCE => :Energy,
+            :MCL => :Loss,
+        ),
+    )
+    return df
+end

src/helpers/zip_helper.jl

@@ -0,0 +1,18 @@
+"""
+    _unzip(filename::AbstractString, func::Function = x -> x)::AbstractString
+
+Unzip all files of 'filename' adding to 'text'.
+"""
+function _unzip(filename::AbstractString, func::Function = x -> x)::Vector{AbstractString}
+    zip = ZipFile.Reader(filename)
+    dest_paths = Vector{AbstractString}()
+    for file in zip.files
+        dest_path = joinpath(dirname(filename), func(file.name))
+        push!(dest_paths, dest_path)
+        if !isfile(dest_path)
+            write(dest_path, read(file))
+        end
+    end
+    close(zip)
+    return dest_paths
+end

test/caiso/datetime.jl

@@ -0,0 +1,7 @@
+@testset "caiso/date.jl" begin
+    @testset "_get_zdt_formated" begin
+        zdt = ZonedDateTime(2023, 5, 4, 7, 2, 13, tz"UTC-3")
+        formated = ElectricityMarketData._get_zdt_formated(zdt)
+        @test formated == "20230504T07:02-0300"
+    end
+end

test/caiso/urls.jl

@@ -0,0 +1,30 @@
+@testset "caiso/urls.jl" begin
+    @testset "_get_caiso_url" begin
+        queryname = "PRC_INTVL_LMP"
+        startdate = "20210105T07:00-0000"
+        stopdate = "20210106T07:00-0000"
+        market_run_id = "RTM"
+        url = ElectricityMarketData._get_caiso_url(
+            queryname,
+            startdate,
+            stopdate,
+            market_run_id,
+        )
+        @test url ==
+              "http://oasis.caiso.com/oasisapi/SingleZip?resultformat=6&queryname=PRC_INTVL_LMP&version=1&startdatetime=20210105T07:00-0000&enddatetime=20210106T07:00-0000&market_run_id=RTM"
+    end
+    @testset "_get_caiso_real_time_url" begin
+        startdate = "20230104T07:00-0000"
+        stopdate = "20230105T07:00-0000"
+        url = ElectricityMarketData._get_caiso_real_time_url(startdate, stopdate)
+        @test url ==
+              "http://oasis.caiso.com/oasisapi/SingleZip?resultformat=6&queryname=PRC_INTVL_LMP&version=1&startdatetime=20230104T07:00-0000&enddatetime=20230105T07:00-0000&market_run_id=RTM"
+    end
+    @testset "_get_caiso_day_ahead_url" begin
+        startdate = "20230104T07:00-0000"
+        stopdate = "20230105T07:00-0000"
+        url = ElectricityMarketData._get_caiso_day_ahead_url(startdate, stopdate)
+        @test url ==
+              "http://oasis.caiso.com/oasisapi/SingleZip?resultformat=6&queryname=PRC_LMP&version=1&startdatetime=20230104T07:00-0000&enddatetime=20230105T07:00-0000&market_run_id=DAM"
+    end
+end

test/helpers/zip_helper.jl

@@ -0,0 +1,13 @@
+@testset "zip_helper.jl" begin
+    @testset "_unzip" begin
+        mktempdir() do tempdir
+            zip = ZipFile.Writer("temp.zip")
+            ZipFile.addfile(zip, "file1.txt")
+            ZipFile.addfile(zip, "file2.txt")
+            close(zip)
+            _ = ElectricityMarketData._unzip("temp.zip", x -> "test_" * x)
+            @test isfile("test_file1.txt")
+            @test isfile("test_file2.txt")
+        end
+    end
+end