.. _T_cell_analysis_label:

T cell analysis with in-data cell sorting
=========================================================

load ECAUGT package
===================

.. code:: ipython3

    # load the ECAUGT package as well as other related packages
    import ECAUGT
    import sys,time,multiprocessing
    import scanpy as sc
    import numpy as np, pandas as pd

.. code:: ipython3

    # set parameters
    endpoint = "https://HCAd-Datasets.cn-beijing.ots.aliyuncs.com"
    access_id = "LTAI5t7t216W9amUD1crMVos" #enter your id and keys
    access_key = "ZJPlUbpLCij5qUPjbsU8GnQHm97IxJ"
    instance_name = "HCAd-Datasets"
    table_name = 'HCA_d'

.. code:: ipython3

    # # setup client
    ECAUGT.Setup_Client(endpoint, access_id, access_key, instance_name, table_name)


.. parsed-literal::

    Connected to the server, find the table.
    HCA_d
    TableName: HCA_d
    PrimaryKey: [('cid', 'INTEGER')]
    Reserved read throughput: 0
    Reserved write throughput: 0
    Last increase throughput time: 1605795297
    Last decrease throughput time: None
    table options's time to live: -1
    table options's max version: 1
    table options's max_time_deviation: 86400




.. parsed-literal::

    0



sorting cells from uGT
======================

sort with labels
----------------

.. code:: ipython3

    query_language = "cell_type == T cell"
    cid_label = ECAUGT.query_cells(metadata_conditions=query_language, include_children=True)



.. parsed-literal::

    51588 cells found


sort with expressional conditions
---------------------------------

.. code:: ipython3

    query_language  = "PTPRC>1.5 && (CD3D>1.5 || CD3E>1.5)"
    gene_condition  = ECAUGT.seq2filter(query_language)
    df_result_tcell = ECAUGT.get_columnsbycell_para(
                                    rows_to_get = None, 
                                    # make sure condition associated columns listed here
                                    cols_to_get=['organ','cell_type','CD3D','CD3E','PTPRC'], 
                                    col_filter=gene_condition, 
                                    do_transfer = True, 
                                    thread_num = 24
                             )


.. parsed-literal::

    1093299 cells found


.. code:: ipython3

    df_result_tcell




.. raw:: html

    <div>
    <style scoped>
        .dataframe tbody tr th:only-of-type {
            vertical-align: middle;
        }
    
        .dataframe tbody tr th {
            vertical-align: top;
        }
    
        .dataframe thead th {
            text-align: right;
        }
    </style>
    <table border="1" class="dataframe">
      <thead>
        <tr style="text-align: right;">
          <th></th>
          <th>CD3D</th>
          <th>CD3E</th>
          <th>PTPRC</th>
          <th>cell_type</th>
          <th>organ</th>
        </tr>
        <tr>
          <th>cid</th>
          <th></th>
          <th></th>
          <th></th>
          <th></th>
          <th></th>
        </tr>
      </thead>
      <tbody>
        <tr>
          <th>35</th>
          <td>2.910235</td>
          <td>2.910235</td>
          <td>3.575773</td>
          <td>T cell</td>
          <td>Spleen</td>
        </tr>
        <tr>
          <th>40</th>
          <td>2.403368</td>
          <td>0.000000</td>
          <td>3.050255</td>
          <td>Neutrophilic granulocyte</td>
          <td>Spleen</td>
        </tr>
        <tr>
          <th>50</th>
          <td>3.820847</td>
          <td>0.000000</td>
          <td>3.149373</td>
          <td>T cell</td>
          <td>Spleen</td>
        </tr>
        <tr>
          <th>126</th>
          <td>0.000000</td>
          <td>4.292039</td>
          <td>3.220413</td>
          <td>T cell</td>
          <td>Spleen</td>
        </tr>
        <tr>
          <th>167</th>
          <td>1.589888</td>
          <td>0.000000</td>
          <td>1.589888</td>
          <td>Plasma B cell</td>
          <td>Spleen</td>
        </tr>
        <tr>
          <th>...</th>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
        </tr>
        <tr>
          <th>4085793</th>
          <td>0.000000</td>
          <td>3.241421</td>
          <td>3.241421</td>
          <td>Zona fasciculata cell</td>
          <td>Adrenal gland</td>
        </tr>
        <tr>
          <th>4089833</th>
          <td>2.782014</td>
          <td>0.000000</td>
          <td>2.782014</td>
          <td>Neutrophilic granulocyte</td>
          <td>Adrenal gland</td>
        </tr>
        <tr>
          <th>4092102</th>
          <td>0.000000</td>
          <td>3.204398</td>
          <td>3.204398</td>
          <td>Neutrophilic granulocyte</td>
          <td>Adrenal gland</td>
        </tr>
        <tr>
          <th>4092673</th>
          <td>2.709732</td>
          <td>0.000000</td>
          <td>2.709732</td>
          <td>Neutrophilic granulocyte</td>
          <td>Adrenal gland</td>
        </tr>
        <tr>
          <th>4093924</th>
          <td>2.256228</td>
          <td>0.000000</td>
          <td>2.256228</td>
          <td>Zona fasciculata cell</td>
          <td>Adrenal gland</td>
        </tr>
      </tbody>
    </table>
    <p>14710 rows × 5 columns</p>
    </div>



.. code:: ipython3

    cid_expression = [[('cid',i)] for i in df_result_tcell.index]

merge two cid obtained from origins
-----------------------------------

.. code:: ipython3

    # merge and remove duplicated cids
    cid_list = set()
    for i in range(len(cid_expression)):
        cid= cid_expression[i][0][1]
        cid_list.add(cid)
        
    for i in range(len(cid_label)):
        cid= cid_label[i][0][1]
        cid_list.add(cid)
        
    cid_list = list(cid_list)
    
    # print number of obtained cids
    print(len(cid_list))
    
    # build rows_to_get variable to download data
    rows_to_get = [[('cid',i)] for i in cid_list]


.. parsed-literal::

    56540


.. code:: ipython3

    import pickle
    pickle.HIGHEST_PROTOCOL




.. parsed-literal::

    4



.. code:: ipython3

    with open('rows_to_get.pickle', 'wb') as f:
        pickle.dump(rows_to_get, f, protocol=4)

.. code:: ipython3

    import pickle
    with open('rows_to_get.pickle', 'rb') as f:
        rows_to_get=pickle.load(f )

.. code:: ipython3

    from tqdm import tqdm
    import pickle
    
    # we suggest downloading cells in small batches in case of network issues
    for chunk in tqdm(range(int(1+len(rows_to_get)/500)),ncols=80):
        # split batches
        lb, rb = chunk*500, (chunk+1)*500
        rows = rows_to_get[lb:rb]
        if len(rows)<=0:break
            
        # download rows from the unified Giant Table (uGT)
        result = ECAUGT.get_columnsbycell_para(rows_to_get = rows, 
                                               cols_to_get = None, # download all columns
                                               col_filter  = None,  
                                               do_transfer = True, 
                                               thread_num  = 24)
        result.to_pickle("__temp_%d_%d.pk"%(lb,rb))
        #print("downloading %d~%d"%(lb, rb))
        #print(len(rows))


.. parsed-literal::

      5%|██▏                                      | 6/114 [09:44<2:55:11, 97.33s/it]OTS request failed, API: GetRow, HTTPStatus: 503, ErrorCode: OTSTimeout, ErrorMessage: Operation timeout., RequestID: 0005d044-f8df-f27c-613f-020a872a9e27.
    100%|███████████████████████████████████████| 114/114 [3:04:25<00:00, 97.06s/it]


.. code:: ipython3

    # load split batches
    giant_table_list = []
    for chunk in tqdm(range(int(1+len(rows_to_get)/500)),ncols=80):
        lb, rb = chunk*500, (chunk+1)*500
        fname = "__temp_%d_%d.pk" % (lb, rb)
        with open(fname,'rb') as f:
            df=pickle.load(f)
            giant_table_list.append(df)



.. parsed-literal::

    100%|█████████████████████████████████████████| 114/114 [00:17<00:00,  6.59it/s]


.. code:: ipython3

    giant_table= giant_table_list[0]
    
    for i in range(1, len(giant_table_list)):
        giant_table = pd.concat([ giant_table, giant_table_list[i] ])


.. code:: ipython3

    # remove intermediate results
    del giant_table_list
    import gc
    gc.collect()
    
    
    giant_table.to_pickle("sorted_tcells_raw.pk")

.. code:: ipython3

    genes    = giant_table.columns[:43878]
    metaCols = giant_table.columns[43878:43878+18]

.. code:: ipython3

    expr = giant_table.loc[:,genes]
    meta = giant_table.loc[:,metaCols]
    
    meta.reset_index(inplace=True)
    expr.reset_index(inplace=True)
    expr=expr.drop(['cid'], axis=1)
    
    print(expr.shape)
    print(meta.shape)


.. parsed-literal::

    (56540, 43878)
    (56540, 19)


.. code:: ipython3

    # check the sample-by-gene expression matrix
    expr




.. raw:: html

    <div>
    <style scoped>
        .dataframe tbody tr th:only-of-type {
            vertical-align: middle;
        }
    
        .dataframe tbody tr th {
            vertical-align: top;
        }
    
        .dataframe thead th {
            text-align: right;
        }
    </style>
    <table border="1" class="dataframe">
      <thead>
        <tr style="text-align: right;">
          <th></th>
          <th>A12M1</th>
          <th>A12M2</th>
          <th>A12M3</th>
          <th>A12M4</th>
          <th>A1BG</th>
          <th>A1BG-AS1</th>
          <th>A1CF</th>
          <th>A2M</th>
          <th>A2M-AS1</th>
          <th>A2ML1</th>
          <th>...</th>
          <th>ZXDA</th>
          <th>ZXDB</th>
          <th>ZXDC</th>
          <th>ZYG11A</th>
          <th>ZYG11AP1</th>
          <th>ZYG11B</th>
          <th>ZYX</th>
          <th>ZYXP1</th>
          <th>ZZEF1</th>
          <th>ZZZ3</th>
        </tr>
      </thead>
      <tbody>
        <tr>
          <th>0</th>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.000000</td>
          <td>...</td>
          <td>0.0</td>
          <td>0.000000</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.000000</td>
          <td>0.0</td>
        </tr>
        <tr>
          <th>1</th>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.000000</td>
          <td>...</td>
          <td>0.0</td>
          <td>0.000000</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.000000</td>
          <td>0.0</td>
        </tr>
        <tr>
          <th>2</th>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.000000</td>
          <td>...</td>
          <td>0.0</td>
          <td>0.000000</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.000000</td>
          <td>0.0</td>
        </tr>
        <tr>
          <th>3</th>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.000000</td>
          <td>...</td>
          <td>0.0</td>
          <td>0.000000</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.000000</td>
          <td>0.0</td>
        </tr>
        <tr>
          <th>4</th>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.000000</td>
          <td>...</td>
          <td>0.0</td>
          <td>0.000000</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.000000</td>
          <td>0.0</td>
        </tr>
        <tr>
          <th>...</th>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
        </tr>
        <tr>
          <th>56535</th>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.000000</td>
          <td>...</td>
          <td>0.0</td>
          <td>0.000000</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.000000</td>
          <td>0.0</td>
        </tr>
        <tr>
          <th>56536</th>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.000000</td>
          <td>...</td>
          <td>0.0</td>
          <td>0.000000</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>2.165166</td>
          <td>0.0</td>
        </tr>
        <tr>
          <th>56537</th>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.000000</td>
          <td>...</td>
          <td>0.0</td>
          <td>2.696737</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.000000</td>
          <td>0.0</td>
        </tr>
        <tr>
          <th>56538</th>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.000000</td>
          <td>...</td>
          <td>0.0</td>
          <td>0.000000</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.000000</td>
          <td>0.0</td>
        </tr>
        <tr>
          <th>56539</th>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>2.658875</td>
          <td>...</td>
          <td>0.0</td>
          <td>0.000000</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.0</td>
          <td>0.000000</td>
          <td>0.0</td>
        </tr>
      </tbody>
    </table>
    <p>56540 rows × 43878 columns</p>
    </div>



.. code:: ipython3

    # check the metadata matrix
    meta




.. raw:: html

    <div>
    <style scoped>
        .dataframe tbody tr th:only-of-type {
            vertical-align: middle;
        }
    
        .dataframe tbody tr th {
            vertical-align: top;
        }
    
        .dataframe thead th {
            text-align: right;
        }
    </style>
    <table border="1" class="dataframe">
      <thead>
        <tr style="text-align: right;">
          <th></th>
          <th>cid</th>
          <th>cell_id</th>
          <th>cell_type</th>
          <th>cl_name</th>
          <th>donor_age</th>
          <th>donor_gender</th>
          <th>donor_id</th>
          <th>hcad_name</th>
          <th>marker_gene</th>
          <th>organ</th>
          <th>original_name</th>
          <th>region</th>
          <th>sample_status</th>
          <th>seq_tech</th>
          <th>study_id</th>
          <th>subregion</th>
          <th>tissue_type</th>
          <th>uHAF_name</th>
          <th>user_id</th>
        </tr>
      </thead>
      <tbody>
        <tr>
          <th>0</th>
          <td>2016314</td>
          <td>ACTGATGCAGCGTTCG-1-HCATisStab7587208</td>
          <td>T cell</td>
          <td>T cell</td>
          <td>NA</td>
          <td>NA</td>
          <td>356C</td>
          <td>Lung-Connective tissue-T cell-CD3D IL32</td>
          <td>CD3D IL32</td>
          <td>Lung</td>
          <td>T_CD4</td>
          <td>Left lung</td>
          <td>Healthy</td>
          <td>10X</td>
          <td>10.1186/s13059-019-1906-x</td>
          <td>Inferior lobular</td>
          <td>Connective tissue</td>
          <td>Lung-Connective tissue-T cell-CD3D IL32</td>
          <td>2</td>
        </tr>
        <tr>
          <th>1</th>
          <td>3100139</td>
          <td>FetalMaleGonad_2.TCACTTGGACATGAGATC</td>
          <td>NK T cell</td>
          <td>NA</td>
          <td>GW11</td>
          <td>Male</td>
          <td>Donor9</td>
          <td>Testis-Connective tissue-NK T cell-MT-ATP6 MT-CYB</td>
          <td>MT-ATP6 MT-CYB</td>
          <td>Testis</td>
          <td>Fetal fibroblast</td>
          <td>NA</td>
          <td>Healthy</td>
          <td>Microwell-seq</td>
          <td>10.1038/s41586-020-2157-4</td>
          <td>NA</td>
          <td>Muscle tissue</td>
          <td>Testis-Connective tissue-NK T cell-MT-ATP6 MT-CYB</td>
          <td>3</td>
        </tr>
        <tr>
          <th>2</th>
          <td>4063239</td>
          <td>AdultGallbladder_2.ACAATAAATAAAGGGCGA3</td>
          <td>T cell</td>
          <td>T cell</td>
          <td>58yr</td>
          <td>Male</td>
          <td>AdultGallbladder2</td>
          <td>Gallbladder-Connective tissue-T cell-IL32</td>
          <td>IL32</td>
          <td>Gallbladder</td>
          <td>T cell_CCL5 high</td>
          <td>NA</td>
          <td>Healthy</td>
          <td>Microwell-seq</td>
          <td>10.1038/s41586-020-2157-4</td>
          <td>NA</td>
          <td>Connective tissue</td>
          <td>Gallbladder-Connective tissue-T cell-IL32</td>
          <td>4</td>
        </tr>
        <tr>
          <th>3</th>
          <td>3100141</td>
          <td>FetalMaleGonad_2.TGATCAGTCCCGAGATGG</td>
          <td>NK T cell</td>
          <td>NA</td>
          <td>GW11</td>
          <td>Male</td>
          <td>Donor9</td>
          <td>Testis-Connective tissue-NK T cell-MT-ATP6 MT-CYB</td>
          <td>MT-ATP6 MT-CYB</td>
          <td>Testis</td>
          <td>Fetal fibroblast</td>
          <td>NA</td>
          <td>Healthy</td>
          <td>Microwell-seq</td>
          <td>10.1038/s41586-020-2157-4</td>
          <td>NA</td>
          <td>Muscle tissue</td>
          <td>Testis-Connective tissue-NK T cell-MT-ATP6 MT-CYB</td>
          <td>3</td>
        </tr>
        <tr>
          <th>4</th>
          <td>4063252</td>
          <td>AdultGallbladder_2.ACAATACATGATCGCACC3</td>
          <td>Epithelial cell</td>
          <td>epithelial cell</td>
          <td>58yr</td>
          <td>Male</td>
          <td>AdultGallbladder2</td>
          <td>Gallbladder-Epithelial tissue-Epithelial cell-...</td>
          <td>TM4SF4</td>
          <td>Gallbladder</td>
          <td>Mucous epithelial cell</td>
          <td>NA</td>
          <td>Healthy</td>
          <td>Microwell-seq</td>
          <td>10.1038/s41586-020-2157-4</td>
          <td>NA</td>
          <td>Epithelial tissue</td>
          <td>Gallbladder-Epithelial tissue-Epithelial cell-...</td>
          <td>4</td>
        </tr>
        <tr>
          <th>...</th>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
        </tr>
        <tr>
          <th>56535</th>
          <td>4063198</td>
          <td>AdultGallbladder_2.AATAAAAGCGAGAGGGTC3</td>
          <td>T cell</td>
          <td>T cell</td>
          <td>58yr</td>
          <td>Male</td>
          <td>AdultGallbladder2</td>
          <td>Gallbladder-Connective tissue-T cell-IL32</td>
          <td>IL32</td>
          <td>Gallbladder</td>
          <td>T cell</td>
          <td>NA</td>
          <td>Healthy</td>
          <td>Microwell-seq</td>
          <td>10.1038/s41586-020-2157-4</td>
          <td>NA</td>
          <td>Connective tissue</td>
          <td>Gallbladder-Connective tissue-T cell-IL32</td>
          <td>4</td>
        </tr>
        <tr>
          <th>56536</th>
          <td>1172316</td>
          <td>FetalMuscle_1.CAACAACCGCTAGGCTGC</td>
          <td>Proliferating T cell</td>
          <td>NA</td>
          <td>GW12</td>
          <td>Male</td>
          <td>NA</td>
          <td>Muscle-Connective tissue-Proliferating T cell-...</td>
          <td>UBE2C</td>
          <td>Muscle</td>
          <td>Proliferating cell_UBE2C high</td>
          <td>NA</td>
          <td>Healthy</td>
          <td>Microwell-seq</td>
          <td>10.1038/s41586-020-2157-4</td>
          <td>NA</td>
          <td>Connective tissue</td>
          <td>Muscle-Connective tissue-Proliferating T cell-...</td>
          <td>1</td>
        </tr>
        <tr>
          <th>56537</th>
          <td>3100133</td>
          <td>FetalMaleGonad_2.TAGCATAACCTACAAAGT</td>
          <td>NK T cell</td>
          <td>NA</td>
          <td>GW11</td>
          <td>Male</td>
          <td>Donor9</td>
          <td>Testis-Connective tissue-NK T cell-MT-ATP6 MT-CYB</td>
          <td>MT-ATP6 MT-CYB</td>
          <td>Testis</td>
          <td>Fetal fibroblast</td>
          <td>NA</td>
          <td>Healthy</td>
          <td>Microwell-seq</td>
          <td>10.1038/s41586-020-2157-4</td>
          <td>NA</td>
          <td>Muscle tissue</td>
          <td>Testis-Connective tissue-NK T cell-MT-ATP6 MT-CYB</td>
          <td>3</td>
        </tr>
        <tr>
          <th>56538</th>
          <td>3100135</td>
          <td>FetalMaleGonad_2.TTTAGGGTGGTACCATCT</td>
          <td>NK T cell</td>
          <td>NA</td>
          <td>GW11</td>
          <td>Male</td>
          <td>Donor9</td>
          <td>Testis-Connective tissue-NK T cell-MT-ATP6 MT-CYB</td>
          <td>MT-ATP6 MT-CYB</td>
          <td>Testis</td>
          <td>Fetal fibroblast</td>
          <td>NA</td>
          <td>Healthy</td>
          <td>Microwell-seq</td>
          <td>10.1038/s41586-020-2157-4</td>
          <td>NA</td>
          <td>Muscle tissue</td>
          <td>Testis-Connective tissue-NK T cell-MT-ATP6 MT-CYB</td>
          <td>3</td>
        </tr>
        <tr>
          <th>56539</th>
          <td>3100136</td>
          <td>FetalMaleGonad_2.CCATCTGCGTCCTGTGCG</td>
          <td>NK T cell</td>
          <td>NA</td>
          <td>GW11</td>
          <td>Male</td>
          <td>Donor9</td>
          <td>Testis-Connective tissue-NK T cell-MT-ATP6 MT-CYB</td>
          <td>MT-ATP6 MT-CYB</td>
          <td>Testis</td>
          <td>Fetal fibroblast</td>
          <td>NA</td>
          <td>Healthy</td>
          <td>Microwell-seq</td>
          <td>10.1038/s41586-020-2157-4</td>
          <td>NA</td>
          <td>Muscle tissue</td>
          <td>Testis-Connective tissue-NK T cell-MT-ATP6 MT-CYB</td>
          <td>3</td>
        </tr>
      </tbody>
    </table>
    <p>56540 rows × 19 columns</p>
    </div>



create single-cell analysis objects
===================================

.. code:: ipython3

    # create scanpy object from the matrices
    adata = sc.AnnData(X = expr, obs = meta)
    adata.var_names_make_unique()
    
    sc.pp.filter_genes(adata, min_counts=5)
    sc.pp.filter_genes(adata, min_cells=3)

.. code:: ipython3

    # post-processing steps
    from scipy.sparse import csc_matrix
    adata.X = csc_matrix(adata.X, dtype=np.float32)
    
    adata.obs['donor_id']=adata.obs['donor_id'].astype(str)

.. code:: ipython3

    adata.write_h5ad("sorted_tcells_raw.h5ad")


.. parsed-literal::

    /home/chensijie/software/anaconda3/envs/r411py37/lib/python3.7/site-packages/anndata/_core/anndata.py:1220: FutureWarning: The `inplace` parameter in pandas.Categorical.reorder_categories is deprecated and will be removed in a future version. Removing unused categories will always return a new Categorical object.
      c.reorder_categories(natsorted(c.categories), inplace=True)
    ... storing 'donor_id' as categorical


filter cells for downstream analysis
====================================

.. code:: ipython3

    # remove fibroblast
    sel = adata[:,"LUM"].X==0
    adata = adata[sel].copy()
    
    sel = adata[:,"SERPING1"].X==0
    adata = adata[sel].copy()
    
    sel = adata[:,"COL1A1"].X==0
    adata = adata[sel].copy()
    
    sel = adata[:,"COL1A2"].X==0
    adata = adata[sel].copy()
    
    # remove vascular endothelial cells
    sel = adata[:,"INMT"].X==0
    adata = adata[sel].copy()
    
    # remove muscle cells
    sel = adata[:,"ACTA2"].X==0
    adata = adata[sel].copy()
    
    # remove granulocytes
    sel = adata[:,"S100A8"].X==0
    adata = adata[sel].copy()
    
    sel = adata[:,"S100A9"].X==0
    adata = adata[sel].copy()
    
    sel = adata[:,"SIGLEC8"].X==0
    adata = adata[sel].copy()
    
    
    # remove myeloid cells
    sel = adata[:,"C1QA"].X==0
    adata = adata[sel].copy()
    
    sel = adata[:,"C1QB"].X==0
    adata = adata[sel].copy()
    
    sel = adata[:,"C1QC"].X==0
    adata = adata[sel].copy()
    
    sel = adata[:,"ITGAM"].X==0
    adata = adata[sel].copy()
    
    sel = adata[:,"ITGAX"].X==0
    adata = adata[sel].copy()
    
    # remove B cells
    sel = adata[:,"CD79A"].X==0
    adata = adata[sel].copy()
    
    sel = adata[:,"CD79B"].X==0
    adata = adata[sel].copy()
    
    sel = adata[:,"CD19"].X==0
    adata = adata[sel].copy()
    
    sel = adata[:,"MS4A1"].X==0
    adata = adata[sel].copy()
    
    # remove cells from nerve tissues
    sel = adata[:,"FSCN1"].X==0
    adata = adata[sel].copy()

.. code:: ipython3

    adata




.. parsed-literal::

    AnnData object with n_obs × n_vars = 20553 × 20491
        obs: 'cid', 'cell_id', 'cell_type', 'cl_name', 'donor_age', 'donor_gender', 'donor_id', 'hcad_name', 'marker_gene', 'organ', 'original_name', 'region', 'sample_status', 'seq_tech', 'study_id', 'subregion', 'tissue_type', 'uHAF_name', 'user_id'
        var: 'n_counts', 'n_cells'



.. code:: ipython3

    sc.pp.highly_variable_genes(adata, min_mean=0.0125, max_mean=3, min_disp=0.5)

.. code:: ipython3

    sc.pl.highly_variable_genes(adata)



.. image:: T_cell_analysis_with_in-data_cell_sorting_files/output_35_0.png


.. code:: ipython3

    sc.tl.pca(adata, svd_solver='arpack')

.. code:: ipython3

    sc.set_figure_params(figsize=[5,5])
    sc.pl.pca(adata,color="organ")



.. image:: T_cell_analysis_with_in-data_cell_sorting_files/output_37_0.png
   :width: 739px
   :height: 357px


.. code:: ipython3

    sc.set_figure_params(figsize=[5,5])
    sc.pl.pca(adata,color="cell_type")



.. image:: T_cell_analysis_with_in-data_cell_sorting_files/output_38_0.png
   :width: 1341px
   :height: 357px


.. code:: ipython3

    sc.set_figure_params(figsize=[5,5])
    sc.pl.pca(adata,color=["PTPRC","CD3D","CD3E","CD3G",
                           "CD8A","CD4","CD69","CCR7"],sort_order=True)



.. image:: T_cell_analysis_with_in-data_cell_sorting_files/output_39_0.png
   :width: 1543px
   :height: 709px


.. code:: ipython3

    adata.write_h5ad("sorted_tcells_filtered.h5ad")

Now we have obtained a collection of T cell using in data sorting. You
can do what ever analysis you like with these sorted cells.

We have done some downstream metabolic pathway analysis on these cells
using GSVA, which is available at
https://github.com/XuegongLab/hECA/tree/main/examples