From c21dd6d2b736a18fa49ccc423184bcf82603de93 Mon Sep 17 00:00:00 2001 From: MireilleLouys Date: Mon, 31 Aug 2026 20:49:13 +0200 Subject: [PATCH 1/4] updates about response tables and appendix B --- Dataproducts-Summary-table.tex | 4 +- HighEnergyObsCoreExt.tex | 19 +++--- Makefile | 2 +- extendedAccessTonewtypesOfproducts.tex | 4 +- ...ndedAccessTonewtypesOfproductsAppendix.tex | 59 +++++++++++++------ 5 files changed, 60 insertions(+), 28 deletions(-) diff --git a/Dataproducts-Summary-table.tex b/Dataproducts-Summary-table.tex index 78ffb54..cb1b4d6 100644 --- a/Dataproducts-Summary-table.tex +++ b/Dataproducts-Summary-table.tex @@ -29,13 +29,13 @@ \subsubsection{Summary Table} {\bf psf} &\raggedright Point Spread Function & A dataset that records the probability density function of spatial/angular spreading of incident particles from a point source caused by the instrument (detector and/or mirror and/or analysis) & \#response-function, \#pdf \cr \noalign{\vspace{8pt}} \hline -\multicolumn{4}{|r|}{\bf Considered for addition to an IVOA Analysis Data Product Vocabulary\footnote{As noted in \S~\ref{sec:dataproduct_type}, we prefer the term ``Advanced Data Product'' rather than ``Analysis Data Product'' since the latter suggests that additional steps ({\em i.e.\/} some type of analysis) have taken place to construct these data products, which may not be the case.}$^,$\footnote{\url{https://github.com/ivoa-std/VEPs/pull/21/changes\#diff-25a867536b53bdb6848baeba09cfc4df1e4ad42da919d440902e57b917dd9abc}.}} \\ \hline +\multicolumn{4}{|r|}{\bf Considered for addition to an IVOA Advanced Data Product Vocabulary} \ \\ \hline \noalign{\vspace{2pt}} {\bf draws} & Draws & A dataset that records statistical draws computed from a probability distribution or a sample population, for example Markov chain Monte Carlo (MCMC) draws used when computing the Bayesian marginal probability density function for a random variable, or the DeltaTS associated with a quantity from a frequentist analysis & none \cr {\bf pdf} &\raggedright Probability Density Function & A dataset that records the probability density function of a quantity, for example the Bayesian marginal probability density function for a random variable & none \cr {\bf region} & Region & A dataset that encodes (one or more) regions of parameter space, for example a spatial region or a region of phase space covered by a dataset. The set of dimensions represented by the region can be arbitrary & none \cr %\sptablerule -\caption{IVOA Vocabulary Extension for High energy data products. } +\caption{IVOA Vocabulary Extension for High energy data products.} \label{tab:dp_vocabulary} \end{longtable} \end{landscape} diff --git a/HighEnergyObsCoreExt.tex b/HighEnergyObsCoreExt.tex index 3a638a2..b5fbd5c 100644 --- a/HighEnergyObsCoreExt.tex +++ b/HighEnergyObsCoreExt.tex @@ -412,21 +412,23 @@ \subsection{Summary} {\centering \bf Column Name} &{\centering \bf UCD} &{\centering \bf Unit} &{\centering \bf Type} &{\centering \bf Description} &{\centering \bf MAN}\\ \hline -{\em ev\_xel\/} & \ucd{meta.number;instr.detection;phys.particle} & unitless & int & {Number of events in an {\bf hea-event-list}}& NO \\ +{\em obs\_publisher\_did} & \ucd{meta.ref.ivoid} &unitless & String & Related dataset ID (foreign key) & YES \\ \hline -{\em s\_ref\_energy\/} & \ucd{meta.ref;em.energy;pos} & eV & float & {Energy at which the ObsCore spatial characterization attributes {\em s\_fov\/} , {\em s\_region\/}, {\em s\_resolution\/} are defined} & NO \\ +{\em ev\_xel\/} & \ucd{meta.number;instr.detection;phys.particle} & unitless & integer & {Number of events in an {\bf hea-event-list}}& NO \\ \hline -{\em em\_ref\_energy\/} & \ucd{meta.ref;em.energy;em} & eV & float & {Energy at which the ObsCore spectral characterization attributes {\em em\_res\_power\/}, {\em em\_resolution\/} are defined} & NO \\ +{\em s\_ref\_energy\/} & \ucd{meta.ref;em.energy;pos} & eV & double & {Energy at which the ObsCore spatial characterization attributes {\em s\_fov\/} , {\em s\_region\/}, {\em s\_resolution\/} are defined} & NO \\ \hline -{\em s\_ref\_oaa\/} & \ucd{pos.posAng;instr.offset;pos} & deg & float & {Off-axis angle ({\em i.e.\/}, the angular separation of the target or source from the telescope optical axis) at which the ObsCore spatial characterization attributes {\em s\_fov\/} , {\em s\_region\/}, {\em s\_resolution\/} are defined} & NO \\ +{\em em\_ref\_energy\/} & \ucd{meta.ref;em.energy;em} & eV & double & {Energy at which the ObsCore spectral characterization attributes {\em em\_res\_power\/}, {\em em\_resolution\/} are defined} & NO \\ \hline -{\em em\_ref\_oaa\/} & \ucd{pos.posAng;instr.offset;em} & deg & float & {Off-axis angle ({\em i.e.\/}, the angular separation of the target or source from the telescope optical axis) at which the ObsCore spectral characterization attributes {\em em\_res\_power\/}, {\em em\_resolution\/} are defined} & NO \\ +{\em s\_ref\_oaa\/} & \ucd{pos.posAng;instr.offset;pos} & deg & double & {Off-axis angle ({\em i.e.\/}, the angular separation of the target or source from the telescope optical axis) at which the ObsCore spatial characterization attributes {\em s\_fov\/} , {\em s\_region\/}, {\em s\_resolution\/} are defined} & NO \\ +\hline +{\em em\_ref\_oaa\/} & \ucd{pos.posAng;instr.offset;em} & deg & double & {Off-axis angle ({\em i.e.\/}, the angular separation of the target or source from the telescope optical axis) at which the ObsCore spectral characterization attributes {\em em\_res\_power\/}, {\em em\_resolution\/} are defined} & NO \\ \hline {\em t\_intervals\/} & \ucd{TBD}& unitless & TMOC & {List of observation intervals or stable/good time intervals describing the exact observation time coverage} & NO \\ \hline -{\em energy\_min\/} & \ucd{em.energy;stat.min} & float & eV & {Energy associated to the ObCcore attribute {\em em\_max\/}, describing the minimal energy of the dataset} & NO \\ +{\em energy\_min\/} & \ucd{em.energy;stat.min} & eV & double & {Energy associated to the ObCcore attribute {\em em\_max\/}, describing the minimal energy of the dataset} & NO \\ \hline -{\em energy\_max\/} & \ucd{em.energy;stat.max} & float & eV & {Energy associated to the ObsCore attribute {\em em\_min\/}, describing the maximal energy of the dataset} & NO \\ +{\em energy\_max\/} & \ucd{em.energy;stat.max} & eV & double & {Energy associated to the ObsCore attribute {\em em\_min\/}, describing the maximal energy of the dataset} & NO \\ \hline {\em obs\_mode\/} & \ucd{meta.code;obs.param} & unitless & string &{Observation mode of the observation ({\em e.g.\/}, TBU)} & NO \\ \hline @@ -526,6 +528,9 @@ \subsubsection{Advanced Data Products} {\bf region}: A dataset that includes an encoding of (one or more) regions of parameter space, for example a spatial region or a region of phase space covered by a dataset. The set of dimensions represented by the region can be arbitrary. \end{quote} +An advanced-dataproduct-type Vocabulary \footnote{\url{https://github.com/ivoa-std/VEPs/pull/21/changes\#diff-25a867536b53bdb6848baeba09cfc4df1e4ad42da919d440902e57b917dd9abc}.} is proposed to the Semantics WG to register these terms in a standard vocabulary, that can be extended to other spectral domains, e.g radio astronomy. + + %mireille proposal for vocabularies \input{Dataproducts-Summary-table} diff --git a/Makefile b/Makefile index c244b42..afcea7a 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ DOCNAME = HighEnergyObsCoreExt DOCVERSION = 1.0 # Publication date, ISO format; update manually for "releases" -DOCDATE = 2026-08-03 +DOCDATE = 2026-08-31 # What is it you're writing: NOTE, WD, PR, REC, PEN, or EN DOCTYPE = PEN diff --git a/extendedAccessTonewtypesOfproducts.tex b/extendedAccessTonewtypesOfproducts.tex index 48e9f02..c9af5b5 100644 --- a/extendedAccessTonewtypesOfproducts.tex +++ b/extendedAccessTonewtypesOfproducts.tex @@ -217,7 +217,9 @@ \subsection{Accessing Datasets via an Alternate Table Joined with the {\tt ivoa. More generally, the cardinality of the relationship between a {\bf response-function} dataset and an {\bf hea-event-list} dataset will vary according to the facility and may be one-to-one, one-to-many, many-to-one, or even many-to-many. Compared to the set of queryable attributes included in {\tt ivoa.obscore} table, typically more but occasionally fewer attributes will be required to select an appropriate {\bf response-function} dataset of a given type. The set of attributes required to identify an appropriate {\bf response-function} dataset will typically depend on the type of {\bf response-function}. -Using this approach, {\bf response-function} data products could be described by one or more alternate {\tt ivoa.response$\{$\_xxx$\}$} tables, where {\tt $\{$\_xxx$\}$} is optional and would depend on the type of {\bf response-function} in the case that different sets of queryable attributes are required for different types of {\bf response-function}s. A conceptual example of a {\tt ivoa.response$\{$\_xxx$\}$} table is presented in Appendix~\ref{sec:accessoptionsappendix}. +Using this approach, {\bf response-function} data products could be described by a {\tt ivoa.response} table . +%{\tt ivoa.response$\{$\_xxx$\}$} tables, where {\tt $\{$\_xxx$\}$} is optional and would depend on the type of {\bf response-function} in the case that different sets of queryable attributes are required for different types of {\bf response-function}s. +A conceptual example of such a {\tt ivoa.response} table is presented in Appendix~\ref{sec:accessoptionsappendix}. A similar strategy could apply for discovering data products for an observation that are associated with an {\bf hea-event-list} data product if those data products require a set of queryable attributes that is not present in the {\tt ivoa.obscore} table. A dedicated table would allow more specific properties of these data products to be listed in optional table columns ({\em e.g.\/}, if certain data products have dependencies on telescope altitude and azimuth, or off-axis and azimuthal angles). Of course, this approach would preclude them from being directly queryable in the {\tt ivoa.obscore} table, which may be undesirable from the user perspective. diff --git a/extendedAccessTonewtypesOfproductsAppendix.tex b/extendedAccessTonewtypesOfproductsAppendix.tex index 26417a0..881b337 100644 --- a/extendedAccessTonewtypesOfproductsAppendix.tex +++ b/extendedAccessTonewtypesOfproductsAppendix.tex @@ -1,11 +1,11 @@ In this appendix we further illustrate the possible use of an alternate table joined with the {\tt ivoa.obscore} table as a method for accessing {\bf response-function}s and data products that require queryable attributes that are not present in the {\tt ivoa.obscore} table, as discussed in \S~\ref{sec:respaccess}. Further scientific input from \gls{HEIG} domain experts, developed from additional use cases that focus on exploring the appropriate set of attributes needed to support such queries, will be needed to further explore and generalize this solution. -In order to handle the various possible cardinality relationships between {\bf response-function} and {\bf hea-event-list} datasets, foreign keys must be defined in the {\tt ivoa.response$\{$\_xxx$\}$} tables that will allow {\tt JOIN} operations between those tables and the {\tt ivoa.obscore} table. +In order to handle the various possible cardinality relationships between {\bf response-function} and {\bf hea-event-list} datasets, foreign keys must be defined in the {\tt ivoa.response} tables that will allow {\tt JOIN} operations between those tables and the {\tt ivoa.obscore} table. + +The {\tt ivoa.response} table uses a specific resp\_dataproduct\_type column which is compliant to the response type vocabulary : \url{https://www.ivoa.net/rdf/response-type/} and has its own resp\_obs\_publisher\_did column as well, in order to distinguish which response matches which ObsCore data set in the {\tt ivoa.response} table. + +\subsection*{Proposal for a Response Table {\tt ivoa.response}} -\subsection*{Example Response Table} -%\TODO{ check fields for this new table } -%\begin{table}[htbp] -%\begin{center} {\small \begin{longtable}{|m{0.25\linewidth}|m{0.1\linewidth}|m{0.10\linewidth}|m{0.41\linewidth}|} \hline @@ -32,15 +32,16 @@ \subsection*{Example Response Table} {\em resp\_energy\_min} & eV & double & Energy band minimal value for response use \\\hline {\em resp\_energy\_max} & eV & double & Energy band maximal value for response use \\\hline -\caption{Example Response Table. With appropriate further study the columns identified herein could form the basis for a recommended base set of columns for any {\tt ivoa.response$\{$\_xxx$\}$} table.} +\caption{Proposal for a standard response table : with appropriate further study the columns identified herein could form the basis for a recommended base set of columns for the {\tt ivoa.response} table.} \label{tab:response_table} \end{longtable} } %\end{center} %\end{table} -\noindent Implementing an {\tt ivoa.response$\{$\_xxx$\}$} table similar to this would allow queries such as the following: \\ - +\subsection*{Example queries} +\noindent Implementing an {\tt ivoa.response} table similar to this would allow queries such as the following: \\ +\subsubsection*{Retrieve psf response for {\em obs\_id} and position criterium } \noindent Find all datasets satisfying: \begin{enumerate}[(i)] \item Position inside 3 arcmin from (83.6324, $+22.0174$), @@ -49,28 +50,52 @@ \subsection*{Example Response Table} \item obs\_collection = ``HESS''. \end{enumerate} -When using the response table defined in Table \ref{tab:response_table}, we can join the {\tt ivoa.obscore} table with the {\tt ivoa.response\_psf} table on +When using the response table defined in Table \ref{tab:response_table}, we can join the {\tt ivoa.obscore} table with the {\tt ivoa.response} table on the two keys {\em obs\_id\/} and {\em obs\_publisher\_did\/} for instance, as in: {\small \begin{verbatim} -SELECT obs_publisher_did, s_ra, s_dec, s_fov, t_min, t_max, +SELECT o.obs_publisher_did, o.obs_id, s_ra, s_dec, s_fov, t_min, t_max, energy_min, energy_max, access_url, access_format, -resp_publisher_did, resp_access_url, resp_access_format, +resp_publisher_did, r.obs_publisher_did, r.obs_id, resp_access_url, resp_access_format, resp_energy_max, resp_energy_min -FROM ivoa.obscore -NATURAL JOIN ivoa.response_psf +FROM ivoa.obscore as o +NATURAL JOIN ivoa.response as r WHERE (target_name = ’Crab’ OR target_name = ’M1’ OR CONTAINS(POINT(s_ra, s_dec), CIRCLE, 83.6324, +22.0174, 0.083333) = 1) AND (resp_product_type = 'psf') -AND (obs_id = '1527') -AND (obs_collection = 'HESS') +AND (o.obs_id = '1527') +AND (o.obs_collection = 'HESS') \end{verbatim} } - -If additional constraints are required, then we can use ({\em e.g.\/}) {\em resp\_t\_min\/}, and {\em resp\_t\_max\/} and constrain the time interval by adding a clause like: +If additional constraints are required, for instance on time, then we can add another set of columns in {\tt ivoa.response}({\em e.g.\/}) {\em resp\_t\_min\/}, and {\em resp\_t\_max\/} and constrain the time interval by adding a clause like: {\small \begin{verbatim} AND (resp_t_min > 56000.0 and resp_t_max < 56001.5) \end{verbatim} } + +\subsubsection*{Retrieve edisp response for data sets constrained by {\em obs\_id} or position and by {\em scan\_mode}} +{\small +\begin{verbatim} +SELECT o.obs_publisher_did, o.obs_id, s_ra, s_dec, s_fov, t_min, t_max, +energy_min, energy_max, access_url, access_format, scan_mode, +resp_publisher_did, resp.obs_id, resp_access_url, resp_access_format, +resp_energy_max, resp_energy_min, +FROM ivoa.response , (SELECT o.obs_publisher_did, o.obs_id, scan_mode +FROM ivoa.obscore +NATURAL JOIN ivoa.obscore-hea +WHERE +(o.obs_collection = 'IACT') +AND (scan_mode=raster-map) ) +AND (o.obs_id = '1976') +OR CONTAINS(POINT(s_ra, s_dec), CIRCLE, 83.6324, +22.0174, 0.083333) = 1) ) as s +WHERE +resp.obs_id =s.obs_id +AND (resp_product_type = 'edisp') +% s is the list of observations satisfying position or obs_id and scan mode criteria +\end{verbatim} +}% end \small + + + From 004a4377af9100f22db64a2d280225b157d828f0 Mon Sep 17 00:00:00 2001 From: Mireille LOUYS <33840665+loumir@users.noreply.github.com> Date: Tue, 1 Sep 2026 17:09:13 +0200 Subject: [PATCH 2/4] Refactor SQL query for response and observation join I tried to clarify the query. The idea is to restrict the search based on the scan_mode column and data set ids and apply the join with response table to a smaller subset . --- ...ndedAccessTonewtypesOfproductsAppendix.tex | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/extendedAccessTonewtypesOfproductsAppendix.tex b/extendedAccessTonewtypesOfproductsAppendix.tex index 881b337..4ffd272 100644 --- a/extendedAccessTonewtypesOfproductsAppendix.tex +++ b/extendedAccessTonewtypesOfproductsAppendix.tex @@ -82,17 +82,17 @@ \subsubsection*{Retrieve edisp response for data sets constrained by {\em obs\_i energy_min, energy_max, access_url, access_format, scan_mode, resp_publisher_did, resp.obs_id, resp_access_url, resp_access_format, resp_energy_max, resp_energy_min, -FROM ivoa.response , (SELECT o.obs_publisher_did, o.obs_id, scan_mode -FROM ivoa.obscore -NATURAL JOIN ivoa.obscore-hea -WHERE -(o.obs_collection = 'IACT') -AND (scan_mode=raster-map) ) -AND (o.obs_id = '1976') -OR CONTAINS(POINT(s_ra, s_dec), CIRCLE, 83.6324, +22.0174, 0.083333) = 1) ) as s -WHERE +FROM ( SELECT o.obs_publisher_did, o.obs_id, scan_mode + FROM ivoa.obscore + NATURAL JOIN ivoa.obscore-hea + WHERE + (o.obs_collection = 'IACT') + AND (scan_mode=raster-map) + AND (o.obs_id = '1976') + OR CONTAINS(POINT(s_ra, s_dec), CIRCLE, 83.6324, +22.0174, 0.083333) = 1) as s +JOIN ivoa.response ON resp.obs_id =s.obs_id -AND (resp_product_type = 'edisp') +WHERE (resp_product_type = 'edisp') % s is the list of observations satisfying position or obs_id and scan mode criteria \end{verbatim} }% end \small From c0b564084ede3f686756b548e2d5e0018fe78e6f Mon Sep 17 00:00:00 2001 From: Mireille LOUYS <33840665+loumir@users.noreply.github.com> Date: Tue, 1 Sep 2026 17:38:35 +0200 Subject: [PATCH 3/4] Update about keys used for join in response table Clarified the joining keys in the response table explanation. --- extendedAccessTonewtypesOfproductsAppendix.tex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extendedAccessTonewtypesOfproductsAppendix.tex b/extendedAccessTonewtypesOfproductsAppendix.tex index 4ffd272..d625cad 100644 --- a/extendedAccessTonewtypesOfproductsAppendix.tex +++ b/extendedAccessTonewtypesOfproductsAppendix.tex @@ -51,7 +51,7 @@ \subsubsection*{Retrieve psf response for {\em obs\_id} and position criterium } \end{enumerate} When using the response table defined in Table \ref{tab:response_table}, we can join the {\tt ivoa.obscore} table with the {\tt ivoa.response} table on -the two keys {\em obs\_id\/} and {\em obs\_publisher\_did\/} for instance, as in: +one of the two keys {\em obs\_id\/} or {\em obs\_publisher\_did\/} for instance, as in: {\small \begin{verbatim} SELECT o.obs_publisher_did, o.obs_id, s_ra, s_dec, s_fov, t_min, t_max, From 6d7014ecca75897ba24ac1d4691b40c74dcd34d8 Mon Sep 17 00:00:00 2001 From: Mireille LOUYS <33840665+loumir@users.noreply.github.com> Date: Tue, 1 Sep 2026 17:41:59 +0200 Subject: [PATCH 4/4] Adjust URL formatting in ivoa.response table section Updated the formatting of the URL in the ivoa.response table description to use a smaller font size. --- extendedAccessTonewtypesOfproductsAppendix.tex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extendedAccessTonewtypesOfproductsAppendix.tex b/extendedAccessTonewtypesOfproductsAppendix.tex index d625cad..9ec0c52 100644 --- a/extendedAccessTonewtypesOfproductsAppendix.tex +++ b/extendedAccessTonewtypesOfproductsAppendix.tex @@ -2,7 +2,7 @@ In order to handle the various possible cardinality relationships between {\bf response-function} and {\bf hea-event-list} datasets, foreign keys must be defined in the {\tt ivoa.response} tables that will allow {\tt JOIN} operations between those tables and the {\tt ivoa.obscore} table. -The {\tt ivoa.response} table uses a specific resp\_dataproduct\_type column which is compliant to the response type vocabulary : \url{https://www.ivoa.net/rdf/response-type/} and has its own resp\_obs\_publisher\_did column as well, in order to distinguish which response matches which ObsCore data set in the {\tt ivoa.response} table. +The {\tt ivoa.response} table uses a specific resp\_dataproduct\_type column which is compliant to the response type vocabulary : \scriptsize{\url{https://www.ivoa.net/rdf/response-type/}} and has its own resp\_obs\_publisher\_did column as well, in order to distinguish which response matches which ObsCore data set in the {\tt ivoa.response} table. \subsection*{Proposal for a Response Table {\tt ivoa.response}}